From d31c520a904a5039bd28bd0c3d0c8bf78f78dd31 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 2 Jan 2024 14:08:41 +0800 Subject: [PATCH] add empty trash api to prune soft deleted items in trash --- packages/api/src/generated/graphql.ts | 42 +++++++++++++++++++++ packages/api/src/generated/schema.graphql | 15 ++++++++ packages/api/src/resolvers/article/index.ts | 24 ++++++++++++ packages/api/src/routers/svc/user.ts | 4 +- packages/api/src/schema.ts | 15 ++++++++ packages/api/src/services/library_item.ts | 27 +++++++++++++ packages/api/src/services/user.ts | 14 ++++--- 7 files changed, 133 insertions(+), 8 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 62ba7be07..c68e340ee 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -727,6 +727,22 @@ export type DeviceTokensSuccess = { deviceTokens: Array; }; +export type EmptyTrashError = { + __typename?: 'EmptyTrashError'; + errorCodes: Array; +}; + +export enum EmptyTrashErrorCode { + Unauthorized = 'UNAUTHORIZED' +} + +export type EmptyTrashResult = EmptyTrashError | EmptyTrashSuccess; + +export type EmptyTrashSuccess = { + __typename?: 'EmptyTrashSuccess'; + success?: Maybe; +}; + export type Feature = { __typename?: 'Feature'; createdAt: Scalars['Date']; @@ -1352,6 +1368,7 @@ export type Mutation = { deleteNewsletterEmail: DeleteNewsletterEmailResult; deleteRule: DeleteRuleResult; deleteWebhook: DeleteWebhookResult; + emptyTrash: EmptyTrashResult; fetchContent: FetchContentResult; generateApiKey: GenerateApiKeyResult; googleLogin: LoginResult; @@ -3663,6 +3680,10 @@ export type ResolversTypes = { DeviceTokensErrorCode: DeviceTokensErrorCode; DeviceTokensResult: ResolversTypes['DeviceTokensError'] | ResolversTypes['DeviceTokensSuccess']; DeviceTokensSuccess: ResolverTypeWrapper; + EmptyTrashError: ResolverTypeWrapper; + EmptyTrashErrorCode: EmptyTrashErrorCode; + EmptyTrashResult: ResolversTypes['EmptyTrashError'] | ResolversTypes['EmptyTrashSuccess']; + EmptyTrashSuccess: ResolverTypeWrapper; Feature: ResolverTypeWrapper; Feed: ResolverTypeWrapper; FeedArticle: ResolverTypeWrapper; @@ -4169,6 +4190,9 @@ export type ResolversParentTypes = { DeviceTokensError: DeviceTokensError; DeviceTokensResult: ResolversParentTypes['DeviceTokensError'] | ResolversParentTypes['DeviceTokensSuccess']; DeviceTokensSuccess: DeviceTokensSuccess; + EmptyTrashError: EmptyTrashError; + EmptyTrashResult: ResolversParentTypes['EmptyTrashError'] | ResolversParentTypes['EmptyTrashSuccess']; + EmptyTrashSuccess: EmptyTrashSuccess; Feature: Feature; Feed: Feed; FeedArticle: FeedArticle; @@ -4975,6 +4999,20 @@ export type DeviceTokensSuccessResolvers; }; +export type EmptyTrashErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type EmptyTrashResultResolvers = { + __resolveType: TypeResolveFn<'EmptyTrashError' | 'EmptyTrashSuccess', ParentType, ContextType>; +}; + +export type EmptyTrashSuccessResolvers = { + success?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type FeatureResolvers = { createdAt?: Resolver; expiresAt?: Resolver, ParentType, ContextType>; @@ -5460,6 +5498,7 @@ export type MutationResolvers>; deleteRule?: Resolver>; deleteWebhook?: Resolver>; + emptyTrash?: Resolver; fetchContent?: Resolver>; generateApiKey?: Resolver>; googleLogin?: Resolver>; @@ -6647,6 +6686,9 @@ export type Resolvers = { DeviceTokensError?: DeviceTokensErrorResolvers; DeviceTokensResult?: DeviceTokensResultResolvers; DeviceTokensSuccess?: DeviceTokensSuccessResolvers; + EmptyTrashError?: EmptyTrashErrorResolvers; + EmptyTrashResult?: EmptyTrashResultResolvers; + EmptyTrashSuccess?: EmptyTrashSuccessResolvers; Feature?: FeatureResolvers; Feed?: FeedResolvers; FeedArticle?: FeedArticleResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index cbcdc1808..5c2431d8a 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -644,6 +644,20 @@ type DeviceTokensSuccess { deviceTokens: [DeviceToken!]! } +type EmptyTrashError { + errorCodes: [EmptyTrashErrorCode!]! +} + +enum EmptyTrashErrorCode { + UNAUTHORIZED +} + +union EmptyTrashResult = EmptyTrashError | EmptyTrashSuccess + +type EmptyTrashSuccess { + success: Boolean +} + type Feature { createdAt: Date! expiresAt: Date @@ -1213,6 +1227,7 @@ type Mutation { deleteNewsletterEmail(newsletterEmailId: ID!): DeleteNewsletterEmailResult! deleteRule(id: ID!): DeleteRuleResult! deleteWebhook(id: ID!): DeleteWebhookResult! + emptyTrash: EmptyTrashResult! fetchContent(id: ID!): FetchContentResult! generateApiKey(input: GenerateApiKeyInput!): GenerateApiKeyResult! googleLogin(input: GoogleLoginInput!): LoginResult! diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 5d698fc81..a1b95ee92 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -21,6 +21,8 @@ import { CreateArticleError, CreateArticleErrorCode, CreateArticleSuccess, + EmptyTrashError, + EmptyTrashSuccess, FetchContentError, FetchContentErrorCode, FetchContentSuccess, @@ -71,6 +73,7 @@ import { findOrCreateLabels, } from '../../services/labels' import { + batchDelete, batchUpdateLibraryItems, createLibraryItem, findLibraryItemById, @@ -1035,6 +1038,27 @@ export const fetchContentResolver = authorized< } }) +export const emptyTrashResolver = authorized< + EmptyTrashSuccess, + EmptyTrashError +>(async (_, __, { uid }) => { + analytics.track({ + userId: uid, + event: 'empty_trash', + }) + + await batchDelete({ + state: LibraryItemState.Deleted, + user: { + id: uid, + }, + }) + + return { + success: true, + } +}) + const getUpdateReason = (libraryItem: LibraryItem, since: Date) => { if (libraryItem.deletedAt) { return UpdateReason.Deleted diff --git a/packages/api/src/routers/svc/user.ts b/packages/api/src/routers/svc/user.ts index 0c7fc5991..cf9519c69 100644 --- a/packages/api/src/routers/svc/user.ts +++ b/packages/api/src/routers/svc/user.ts @@ -5,7 +5,7 @@ import express from 'express' import { LessThan } from 'typeorm' import { StatusType } from '../../entity/user' import { readPushSubscription } from '../../pubsub' -import { batchDeleteUsers } from '../../services/user' +import { batchDelete } from '../../services/user' import { corsConfig } from '../../utils/corsConfig' import { logger } from '../../utils/logger' @@ -52,7 +52,7 @@ export function userServiceRouter() { const subTime = cleanupMessage.subDays * 1000 * 60 * 60 * 24 // convert days to milliseconds try { - const result = await batchDeleteUsers({ + const result = await batchDelete({ status: StatusType.Deleted, updatedAt: LessThan(new Date(Date.now() - subTime)), // subDays ago }) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4df9a5b35..47720b676 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2761,6 +2761,20 @@ const schema = gql` BAD_REQUEST } + union EmptyTrashResult = EmptyTrashSuccess | EmptyTrashError + + type EmptyTrashSuccess { + success: Boolean + } + + type EmptyTrashError { + errorCodes: [EmptyTrashErrorCode!]! + } + + enum EmptyTrashErrorCode { + UNAUTHORIZED + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2872,6 +2886,7 @@ const schema = gql` updateNewsletterEmail( input: UpdateNewsletterEmailInput! ): UpdateNewsletterEmailResult! + emptyTrash: EmptyTrashResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index fdfb4e962..9aca2c098 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1073,3 +1073,30 @@ export const deleteLibraryItemsByAdmin = async ( 'admin' ) } + +export const batchDelete = async (criteria: FindOptionsWhere) => { + const batchSize = 1000 + + const qb = libraryItemRepository.createQueryBuilder().where(criteria) + const countSql = queryBuilderToRawSql(qb.select('COUNT(1)')) + const subQuery = queryBuilderToRawSql(qb.select('id').limit(batchSize)) + + const sql = ` + -- Set batch size + DO $$ + DECLARE + batch_size INT := ${batchSize}; + BEGIN + -- Loop through batches + FOR i IN 0..CEIL((${countSql})) * 1.0 / batch_size) - 1 LOOP + -- Delete batch + DELETE FROM omnivore.library_item + WHERE id = ANY( + ${subQuery} + ); + END LOOP; + END $$ + ` + + return authTrx(async (t) => t.query(sql)) +} diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts index 848c12b0c..4c8cabbd7 100644 --- a/packages/api/src/services/user.ts +++ b/packages/api/src/services/user.ts @@ -48,13 +48,12 @@ export const createUsers = async (users: DeepPartial[]) => { ) } -export const batchDeleteUsers = async (criteria: FindOptionsWhere) => { +export const batchDelete = async (criteria: FindOptionsWhere) => { const userQb = getRepository(User).createQueryBuilder().where(criteria) const userCountSql = queryBuilderToRawSql(userQb.select('COUNT(1)')) const userSubQuery = queryBuilderToRawSql(userQb.select('id INTO user_ids')) const batchSize = 1000 - const start = new Date().toISOString() const sql = ` -- Set batch size DO $$ @@ -65,15 +64,18 @@ export const batchDeleteUsers = async (criteria: FindOptionsWhere) => { -- Loop through batches of users FOR i IN 0..CEIL((${userCountSql})) * 1.0 / batch_size) - 1 LOOP -- GET batch of user ids - ${userSubQuery} LIMIT ${batchSize} OFFSET i * batch_size; + ${userSubQuery} LIMIT batch_size OFFSET i * batch_size; -- Loop through batches of items FOR j IN 0..CEIL((SELECT COUNT(1) FROM omnivore.library_item WHERE user_id = ANY(user_ids))) * 1.0 / batch_size) - 1 LOOP -- Delete batch of items DELETE FROM omnivore.library_item - WHERE user_id = ANY(user_ids) - AND updated_at < '${start}' - LIMIT ${batchSize}; + WHERE id = ANY( + SELECT id + FROM omnivore.library_item + WHERE user_id = ANY(user_ids) + LIMIT batch_size + ); END LOOP; -- Delete the batch of users