add tiktok

package.json

@@ -74,6 +74,7 @@
     "moment-timezone": "^0.5.43",
     "morgan": "^1.10.0",
     "multer": "^1.4.5-lts.1",
+    "mysql2": "^3.11.5",
     "nanoid": "3",
     "node-fetch": "^3.3.2",
     "node-global-proxy": "^1.0.1",

src/controllers/index.ts

@@ -3,12 +3,14 @@ import XController from './x'
 import TweetController from './tweet'
 import ScoreController from './score'
 import ArtistController from './artist'
+import TiktokController from './tiktok'
 export * from './types'
 
 export {
   PingpongController,
   XController,
   TweetController,
+  TiktokController,
   ScoreController,
   ArtistController,
 }

src/controllers/tiktok/index.ts

@@ -0,0 +1,175 @@
+import { Request, RequestHandler, NextFunction, Router } from 'express'
+import { Controller } from '../types'
+import jsonResponseMiddleware, {
+  JsonResponse,
+} from '../../middleware/jsonResponse.middleware'
+import { PaginationConnection } from 'sequelize-cursor-pagination'
+import { MODIFY_TOKEN } from '../../constants'
+import tiktokService, {
+  GenerateTiktokResult,
+  ShotData,
+} from '../../services/tiktokService'
+import { TiktokData } from '../../db/models/Tiktok'
+
+interface ImportPayload {
+  token: string
+  urls: string[]
+}
+
+interface ImportShotPayload {
+  token: string
+  shots: ShotData[]
+}
+
+interface TopPayload {
+  token: string
+  statusId: string
+  top?: boolean
+}
+
+interface DeletePayload {
+  token: string
+  statusId: string
+}
+
+export default class TiktokController implements Controller {
+  public path = '/api/v1/tiktok'
+  public router = Router()
+
+  constructor() {
+    this.initializeRoutes()
+  }
+
+  private initializeRoutes(): void {
+    this.router.get(
+      '/list',
+      // apiKeyMiddleware(),
+      jsonResponseMiddleware,
+      this.list as RequestHandler
+    )
+    this.router.post(
+      '/import',
+      // apiKeyMiddleware(),
+      jsonResponseMiddleware,
+      this.import as RequestHandler
+    )
+    this.router.post(
+      '/import_shot',
+      // apiKeyMiddleware(),
+      jsonResponseMiddleware,
+      this.importShot as RequestHandler
+    )
+    this.router.post(
+      '/top',
+      // apiKeyMiddleware(),
+      jsonResponseMiddleware,
+      this.top as RequestHandler
+    )
+    this.router.post(
+      '/delete',
+      // apiKeyMiddleware(),
+      jsonResponseMiddleware,
+      this.delete as RequestHandler
+    )
+  }
+
+  private list(
+    request: Request<any, any, any, { after?: string }>,
+    response: JsonResponse<PaginationConnection<TiktokData>>,
+    next: NextFunction
+  ): void {
+    const after = request.query.after
+    tiktokService
+      .paginateTweetByOrder(after)
+      .then((tiktok) => {
+        response.jsonSuccess(tiktok)
+      })
+      .catch((e) => {
+        response.status(500).jsonError('Server Error', 1010)
+      })
+  }
+
+  private import(
+    request: Request<any, any, ImportPayload>,
+    response: JsonResponse<GenerateTiktokResult[]>,
+    next: NextFunction
+  ): void {
+    const { urls, token } = request.body
+    if (token !== MODIFY_TOKEN) {
+      response.status(401).jsonError('Unauthorized', 1012)
+      return
+    }
+
+    tiktokService
+      .bulkGenerateTiktokByPublishAPI(urls)
+      .then((tiktoks) => {
+        response.jsonSuccess(tiktoks)
+      })
+      .catch((e) => {
+        response.status(500).jsonError('Server Error', 1011)
+      })
+  }
+
+  private importShot(
+    request: Request<any, any, ImportShotPayload>,
+    response: JsonResponse<GenerateTiktokResult[]>,
+    next: NextFunction
+  ): void {
+    const { shots, token } = request.body
+    if (token !== MODIFY_TOKEN) {
+      response.status(401).jsonError('Unauthorized', 1012)
+      return
+    }
+
+    tiktokService
+      .bulkImportShot(shots)
+      .then((tweets) => {
+        response.jsonSuccess(tweets)
+      })
+      .catch((e) => {
+        response.status(500).jsonError('Server Error', 1011)
+      })
+  }
+
+  private top(
+    request: Request<any, any, TopPayload>,
+    response: JsonResponse<boolean>,
+    next: NextFunction
+  ): void {
+    const { statusId, top, token } = request.body
+    if (token !== MODIFY_TOKEN) {
+      response.status(401).jsonError('Unauthorized', 1012)
+      return
+    }
+
+    tiktokService
+      .makeTweetTop(statusId, top)
+      .then(() => {
+        response.jsonSuccess(true)
+      })
+      .catch((e) => {
+        response.status(500).jsonError('Server Error', 1014)
+      })
+  }
+
+  private delete(
+    request: Request<any, any, DeletePayload>,
+    response: JsonResponse<boolean>,
+    next: NextFunction
+  ): void {
+    const { statusId, token } = request.body
+    if (token !== MODIFY_TOKEN) {
+      response.status(401).jsonError('Unauthorized', 1012)
+      return
+    }
+
+    tiktokService
+      .makeTweetDisabled(statusId)
+      .then(() => {
+        response.jsonSuccess(true)
+      })
+      .catch((e) => {
+        response.status(500).jsonError('Server Error', 1014)
+      })
+  }
+}

src/db/index.ts

@@ -4,13 +4,13 @@ import { RPC } from '../utils/EVMHelper/dbModels'
 import { DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASS } from '../constants'
 // import { promisify } from 'util'
 
-const { Constant, Application, Tweet, Score, Art } = Models
+const { Constant, Application, Tweet, Score, Art, Tiktok } = Models
 
 const sequelize = new Sequelize({
   host: DB_HOST,
   port: DB_PORT ? Number.parseInt(DB_PORT) : 3306,
   database: DB_NAME,
-  dialect: 'mariadb',
+  dialect: 'mysql',
   username: DB_USER,
   password: DB_PASS,
   dialectOptions: {
@@ -21,7 +21,7 @@ const sequelize = new Sequelize({
   pool: {
     max: 25,
   },
-  models: [Constant, RPC, Application, Tweet, Score, Art],
+  models: [Constant, RPC, Application, Tweet, Score, Art, Tiktok],
 })
 
 export default sequelize

src/db/models/Tiktok.ts

@@ -0,0 +1,92 @@
+import {
+  Table,
+  Column,
+  AllowNull,
+  DataType,
+  Model,
+  Default,
+} from 'sequelize-typescript'
+
+export interface TiktokData {
+  username: string
+  statusId: string
+  url: string
+  cleanHTML: string
+  body: object
+  order: number
+  type: number
+}
+
+@Table({
+  modelName: 'tiktok',
+  indexes: [
+    {
+      fields: ['statusId'],
+      unique: true,
+    },
+  ],
+})
+export default class Tiktok extends Model {
+  @AllowNull(false)
+  @Column(DataType.CHAR(255))
+  get username(): string {
+    return this.getDataValue('username')
+  }
+
+  @AllowNull(false)
+  @Column(DataType.CHAR(30))
+  get statusId(): string {
+    return this.getDataValue('statusId')
+  }
+
+  @AllowNull(false)
+  @Column(DataType.TEXT)
+  get url(): string {
+    return this.getDataValue('url')
+  }
+
+  @AllowNull(false)
+  @Column(DataType.TEXT)
+  get cleanHTML(): string {
+    return this.getDataValue('cleanHTML')
+  }
+
+  @AllowNull(false)
+  @Default(false)
+  @Column(DataType.BOOLEAN)
+  get disabled(): boolean {
+    return this.getDataValue('disabled')
+  }
+
+  @AllowNull(false)
+  @Column(DataType.JSON)
+  get body(): object {
+    return this.getDataValue('body')
+  }
+
+  @AllowNull(false)
+  @Default(0)
+  @Column(DataType.INTEGER.UNSIGNED)
+  get type(): number {
+    return this.getDataValue('order')
+  }
+
+  @AllowNull(false)
+  @Default(0)
+  @Column(DataType.INTEGER.UNSIGNED)
+  get order(): number {
+    return this.getDataValue('order')
+  }
+
+  getData(): TiktokData {
+    return {
+      username: this.username,
+      statusId: this.statusId,
+      url: this.url,
+      cleanHTML: this.cleanHTML,
+      body: this.body,
+      order: this.order,
+      type: this.type,
+    }
+  }
+}

src/db/models/index.ts

@@ -3,5 +3,6 @@ import Application from './Application'
 import Tweet from './Tweet'
 import Score from './Score'
 import Art from './Art'
+import Tiktok from './Tiktok'
 
-export { Constant, Application, Tweet, Score, Art }
+export { Constant, Application, Tweet, Score, Art, Tiktok }

src/services/tiktokService/index.ts

@@ -0,0 +1,321 @@
+import { Tiktok } from '../../db/models'
+import axios from 'axios'
+import {
+  makePaginate,
+  PaginateOptions,
+  PaginationConnection,
+} from 'sequelize-cursor-pagination'
+import { TweetData } from '../../db/models/Tweet'
+import { TiktokData } from '../../db/models/Tiktok'
+
+const STATE = 'banana'
+
+const tweetUrlReg = /^https:\/\/x\.com\/[a-zA-Z0-9_]+\/status\/([0-9]+)$/
+const tiktokUrlReg = /^https:\/\/www\.tiktok\.com\/@[a-zA-Z0-9_\-]+\/video\/([0-9]+)$/
+
+const removeScriptReg = /<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi
+
+async function generateTweetByPublishAPI(urlRaw: string): Promise<Tiktok> {
+  const url = urlRaw.trim().split('?')[0]
+  const match = url.match(tweetUrlReg)
+  if (!match) {
+    throw new Error('Invalid tweet URL')
+  }
+
+  const statusId = match[1]
+
+  try {
+    const resp = await axios.get('https://publish.twitter.com/oembed', {
+      params: {
+        url,
+        partner: '',
+        hideConversation: 'on',
+        hide_thread: 'true',
+        omit_script: 'false',
+      },
+    })
+
+    const data = resp.data
+    if (!data.html || !data.author_name) {
+      throw new Error('Failed to fetch tweet HTML')
+    }
+    const cleanHTML = data.html.replace(removeScriptReg, '').trim()
+
+    const [tiktok, created] = await Tiktok.findOrCreate({
+      where: {
+        statusId,
+      },
+      defaults: {
+        statusId,
+        username: data.author_name,
+        url,
+        body: data,
+        cleanHTML,
+        type: 0,
+      },
+    })
+
+    if (!created) {
+      tiktok.setDataValue('username', data.author_name)
+      tiktok.setDataValue('url', url)
+      tiktok.setDataValue('body', data)
+      tiktok.setDataValue('cleanHTML', cleanHTML)
+      tiktok.setDataValue('disabled', false)
+      tiktok.setDataValue('type', 0)
+      await tiktok.save()
+    }
+
+    return tiktok
+  } catch (error: any) {
+    if (error.response?.data?.error) {
+      throw new Error(error.response.data.error)
+    }
+    throw error
+  }
+}
+async function generateTiktokByPublishAPI(urlRaw: string): Promise<Tiktok> {
+  const url = urlRaw.trim().split('?')[0]
+  const match = url.match(tiktokUrlReg)
+  if (!match) {
+    throw new Error('Invalid tiktok URL')
+  }
+
+  const statusId = match[1]
+
+  try {
+    const resp = await axios.get('https://www.tiktok.com/oembed', {
+      params: {
+        url,
+      },
+    })
+
+    const data = resp.data
+    if (!data.html || !data.author_name) {
+      throw new Error('Failed to fetch tiktok HTML')
+    }
+    const cleanHTML = data.html.replace(removeScriptReg, '').trim()
+
+    const [tiktok, created] = await Tiktok.findOrCreate({
+      where: {
+        statusId,
+      },
+      defaults: {
+        statusId,
+        username: data.author_name,
+        url,
+        body: data,
+        cleanHTML,
+        type: 1,
+      },
+    })
+
+    if (!created) {
+      tiktok.setDataValue('username', data.author_name)
+      tiktok.setDataValue('url', url)
+      tiktok.setDataValue('body', data)
+      tiktok.setDataValue('cleanHTML', cleanHTML)
+      tiktok.setDataValue('disabled', false)
+      tiktok.setDataValue('type', 1)
+      await tiktok.save()
+    }
+
+    return tiktok
+  } catch (error: any) {
+    if (error.response?.data?.error) {
+      throw new Error(error.response.data.error)
+    }
+    throw error
+  }
+}
+
+// console.log(Tiktok.rawAttributes)
+let paginate: (
+  this: unknown,
+  queryOptions: PaginateOptions<Tiktok>
+) => Promise<PaginationConnection<Tiktok>>
+
+async function paginateTweetByOrder(
+  after?: string,
+  limit = 24
+): Promise<PaginationConnection<TiktokData>> {
+  if (!paginate) {
+    paginate = makePaginate(Tiktok)
+  }
+
+  const secondResult = await paginate({
+    order: [
+      ['order', 'DESC'],
+      ['id', 'DESC'],
+    ],
+    limit,
+    after,
+    where: {
+      disabled: false,
+    },
+  })
+
+  const ret: PaginationConnection<TiktokData> = {
+    ...secondResult,
+    edges: secondResult.edges.map((edge) => ({
+      cursor: edge.node.id,
+      node: edge.node.getData(),
+    })),
+  }
+
+  return ret
+}
+
+export interface GenerateTiktokResult {
+  url: string
+  success: boolean
+  error?: string
+}
+
+async function bulkGenerateTiktokByPublishAPI(
+  urls: string[]
+): Promise<GenerateTiktokResult[]> {
+  const results: GenerateTiktokResult[] = []
+  for (const url of urls) {
+    try {
+      if (url.includes('tiktok')) {
+        await generateTiktokByPublishAPI(url)
+        results.push({ url, success: true })
+      } else if (url.includes('x.com')) {
+        await generateTweetByPublishAPI(url)
+        results.push({ url, success: true })
+      } else {
+        results.push({
+          url,
+          success: false,
+          error: 'Invalid URL',
+        })
+      }
+    } catch (e) {
+      results.push({
+        url,
+        success: false,
+        error: (e as Error).message ?? 'unknown error',
+      })
+    }
+  }
+  return results
+}
+
+async function makeTweetTop(statusId: string, top = true): Promise<void> {
+  const tweet = await Tiktok.findOne({ where: { statusId } })
+  if (!tweet) {
+    throw new Error('Tiktok not found')
+  }
+  if (top) {
+    const maxOrder: number = await Tiktok.max('order')
+    tweet.setDataValue('order', maxOrder ? maxOrder + 1 : 1)
+  } else {
+    tweet.setDataValue('order', 0)
+  }
+  await tweet.save()
+}
+
+async function makeTweetDisabled(
+  statusId: string,
+  disabled = true
+): Promise<void> {
+  const tweet = await Tiktok.findOne({ where: { statusId } })
+  if (!tweet) {
+    throw new Error('Tiktok not found')
+  }
+  tweet.setDataValue('disabled', disabled)
+  await tweet.save()
+}
+
+export interface ShotData {
+  url: string
+  shot: string
+}
+
+async function importShot(data: ShotData): Promise<Tiktok> {
+  const { shot: shotRaw, url: urlRaw } = data
+
+  try {
+    return await generateTweetByPublishAPI(urlRaw)
+  } catch (error) {
+    // if error, use shot
+  }
+
+  const shot = shotRaw.trim()
+
+  if (!shot?.startsWith('https://')) {
+    throw new Error('Invalid shot image URL')
+  }
+
+  const url = urlRaw.trim().split('?')[0]
+  const match = url.match(tweetUrlReg)
+  if (!match) {
+    throw new Error('Invalid tweet URL')
+  }
+
+  const statusId = match[1]
+
+  try {
+    const [tweet, created] = await Tiktok.findOrCreate({
+      where: {
+        statusId,
+      },
+      defaults: {
+        statusId,
+        username: '',
+        url,
+        body: data,
+        cleanHTML: '',
+      },
+    })
+
+    if (!created) {
+      tweet.setDataValue('username', '')
+      tweet.setDataValue('url', url)
+      tweet.setDataValue('body', data)
+      tweet.setDataValue('cleanHTML', '')
+      tweet.setDataValue('disabled', false)
+      tweet.setDataValue('type', 0)
+      await tweet.save()
+    }
+
+    return tweet
+  } catch (error: any) {
+    if (error.response?.data?.error) {
+      throw new Error(error.response.data.error)
+    }
+    throw error
+  }
+}
+
+async function bulkImportShot(
+  datas: ShotData[]
+): Promise<GenerateTiktokResult[]> {
+  const results: GenerateTiktokResult[] = []
+  for (const data of datas) {
+    try {
+      await importShot(data)
+      results.push({ url: data.url, success: true })
+    } catch (e) {
+      results.push({
+        url: data.url,
+        success: false,
+        error: (e as Error).message ?? 'unknown error',
+      })
+    }
+  }
+  return results
+}
+
+const tiktokService = {
+  STATE,
+  generateTweetByPublishAPI,
+  paginateTweetByOrder,
+  bulkGenerateTiktokByPublishAPI,
+  makeTweetTop,
+  makeTweetDisabled,
+  importShot,
+  bulkImportShot,
+}
+
+export default tiktokService

src/start.ts

@@ -4,6 +4,7 @@ import {
   PingpongController,
   XController,
   TweetController,
+  TiktokController,
   ScoreController,
   ArtistController,
 } from './controllers'
@@ -16,6 +17,7 @@ const app = new APP({
   controllers: [
     new PingpongController(),
     new XController(),
+    new TiktokController(),
     new TweetController(),
     new ScoreController(),
     new ArtistController(),