diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 62ba7be07..c68e340ee 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -727,6 +727,22 @@ export type DeviceTokensSuccess = { deviceTokens: Array; }; +export type EmptyTrashError = { + __typename?: 'EmptyTrashError'; + errorCodes: Array; +}; + +export enum EmptyTrashErrorCode { + Unauthorized = 'UNAUTHORIZED' +} + +export type EmptyTrashResult = EmptyTrashError | EmptyTrashSuccess; + +export type EmptyTrashSuccess = { + __typename?: 'EmptyTrashSuccess'; + success?: Maybe; +}; + export type Feature = { __typename?: 'Feature'; createdAt: Scalars['Date']; @@ -1352,6 +1368,7 @@ export type Mutation = { deleteNewsletterEmail: DeleteNewsletterEmailResult; deleteRule: DeleteRuleResult; deleteWebhook: DeleteWebhookResult; + emptyTrash: EmptyTrashResult; fetchContent: FetchContentResult; generateApiKey: GenerateApiKeyResult; googleLogin: LoginResult; @@ -3663,6 +3680,10 @@ export type ResolversTypes = { DeviceTokensErrorCode: DeviceTokensErrorCode; DeviceTokensResult: ResolversTypes['DeviceTokensError'] | ResolversTypes['DeviceTokensSuccess']; DeviceTokensSuccess: ResolverTypeWrapper; + EmptyTrashError: ResolverTypeWrapper; + EmptyTrashErrorCode: EmptyTrashErrorCode; + EmptyTrashResult: ResolversTypes['EmptyTrashError'] | ResolversTypes['EmptyTrashSuccess']; + EmptyTrashSuccess: ResolverTypeWrapper; Feature: ResolverTypeWrapper; Feed: ResolverTypeWrapper; FeedArticle: ResolverTypeWrapper; @@ -4169,6 +4190,9 @@ export type ResolversParentTypes = { DeviceTokensError: DeviceTokensError; DeviceTokensResult: ResolversParentTypes['DeviceTokensError'] | ResolversParentTypes['DeviceTokensSuccess']; DeviceTokensSuccess: DeviceTokensSuccess; + EmptyTrashError: EmptyTrashError; + EmptyTrashResult: ResolversParentTypes['EmptyTrashError'] | ResolversParentTypes['EmptyTrashSuccess']; + EmptyTrashSuccess: EmptyTrashSuccess; Feature: Feature; Feed: Feed; FeedArticle: FeedArticle; @@ -4975,6 +4999,20 @@ export type DeviceTokensSuccessResolvers; }; +export type EmptyTrashErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type EmptyTrashResultResolvers = { + __resolveType: TypeResolveFn<'EmptyTrashError' | 'EmptyTrashSuccess', ParentType, ContextType>; +}; + +export type EmptyTrashSuccessResolvers = { + success?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type FeatureResolvers = { createdAt?: Resolver; expiresAt?: Resolver, ParentType, ContextType>; @@ -5460,6 +5498,7 @@ export type MutationResolvers>; deleteRule?: Resolver>; deleteWebhook?: Resolver>; + emptyTrash?: Resolver; fetchContent?: Resolver>; generateApiKey?: Resolver>; googleLogin?: Resolver>; @@ -6647,6 +6686,9 @@ export type Resolvers = { DeviceTokensError?: DeviceTokensErrorResolvers; DeviceTokensResult?: DeviceTokensResultResolvers; DeviceTokensSuccess?: DeviceTokensSuccessResolvers; + EmptyTrashError?: EmptyTrashErrorResolvers; + EmptyTrashResult?: EmptyTrashResultResolvers; + EmptyTrashSuccess?: EmptyTrashSuccessResolvers; Feature?: FeatureResolvers; Feed?: FeedResolvers; FeedArticle?: FeedArticleResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index cbcdc1808..5c2431d8a 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -644,6 +644,20 @@ type DeviceTokensSuccess { deviceTokens: [DeviceToken!]! } +type EmptyTrashError { + errorCodes: [EmptyTrashErrorCode!]! +} + +enum EmptyTrashErrorCode { + UNAUTHORIZED +} + +union EmptyTrashResult = EmptyTrashError | EmptyTrashSuccess + +type EmptyTrashSuccess { + success: Boolean +} + type Feature { createdAt: Date! expiresAt: Date @@ -1213,6 +1227,7 @@ type Mutation { deleteNewsletterEmail(newsletterEmailId: ID!): DeleteNewsletterEmailResult! deleteRule(id: ID!): DeleteRuleResult! deleteWebhook(id: ID!): DeleteWebhookResult! + emptyTrash: EmptyTrashResult! fetchContent(id: ID!): FetchContentResult! generateApiKey(input: GenerateApiKeyInput!): GenerateApiKeyResult! googleLogin(input: GoogleLoginInput!): LoginResult! diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 6f8ff71e0..08f0401a1 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -21,6 +21,8 @@ import { CreateArticleError, CreateArticleErrorCode, CreateArticleSuccess, + EmptyTrashError, + EmptyTrashSuccess, FetchContentError, FetchContentErrorCode, FetchContentSuccess, @@ -71,6 +73,7 @@ import { findOrCreateLabels, } from '../../services/labels' import { + batchDelete, batchUpdateLibraryItems, createLibraryItem, findLibraryItemById, @@ -1034,6 +1037,27 @@ export const fetchContentResolver = authorized< } }) +export const emptyTrashResolver = authorized< + EmptyTrashSuccess, + EmptyTrashError +>(async (_, __, { uid }) => { + analytics.track({ + userId: uid, + event: 'empty_trash', + }) + + await batchDelete({ + state: LibraryItemState.Deleted, + user: { + id: uid, + }, + }) + + return { + success: true, + } +}) + const getUpdateReason = (libraryItem: LibraryItem, since: Date) => { if (libraryItem.deletedAt) { return UpdateReason.Deleted diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 7c4616ea1..6cff177ca 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -38,6 +38,7 @@ import { generateDownloadSignedUrl, generateUploadFilePathName, } from '../utils/uploads' +import { emptyTrashResolver } from './article' import { optInFeatureResolver } from './features' import { uploadImportFileResolver } from './importers/uploadImportFileResolver' import { @@ -231,6 +232,7 @@ export const functionResolvers = { updateEmail: updateEmailResolver, moveToFolder: moveToFolderResolver, updateNewsletterEmail: updateNewsletterEmailResolver, + emptyTrash: emptyTrashResolver, }, Query: { me: getMeUserResolver, @@ -578,4 +580,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('ScanFeeds'), ...resultResolveTypeResolver('MoveToFolder'), ...resultResolveTypeResolver('UpdateNewsletterEmail'), + ...resultResolveTypeResolver('EmptyTrash'), } diff --git a/packages/api/src/routers/svc/user.ts b/packages/api/src/routers/svc/user.ts index ca6e9dbea..cf9519c69 100644 --- a/packages/api/src/routers/svc/user.ts +++ b/packages/api/src/routers/svc/user.ts @@ -5,7 +5,7 @@ import express from 'express' import { LessThan } from 'typeorm' import { StatusType } from '../../entity/user' import { readPushSubscription } from '../../pubsub' -import { deleteUsers } from '../../services/user' +import { batchDelete } from '../../services/user' import { corsConfig } from '../../utils/corsConfig' import { logger } from '../../utils/logger' @@ -52,7 +52,7 @@ export function userServiceRouter() { const subTime = cleanupMessage.subDays * 1000 * 60 * 60 * 24 // convert days to milliseconds try { - const result = await deleteUsers({ + const result = await batchDelete({ status: StatusType.Deleted, updatedAt: LessThan(new Date(Date.now() - subTime)), // subDays ago }) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4df9a5b35..47720b676 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2761,6 +2761,20 @@ const schema = gql` BAD_REQUEST } + union EmptyTrashResult = EmptyTrashSuccess | EmptyTrashError + + type EmptyTrashSuccess { + success: Boolean + } + + type EmptyTrashError { + errorCodes: [EmptyTrashErrorCode!]! + } + + enum EmptyTrashErrorCode { + UNAUTHORIZED + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2872,6 +2886,7 @@ const schema = gql` updateNewsletterEmail( input: UpdateNewsletterEmailInput! ): UpdateNewsletterEmailResult! + emptyTrash: EmptyTrashResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index c09411943..d6944af12 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1059,3 +1059,30 @@ export const deleteLibraryItemsByAdmin = async ( 'admin' ) } + +export const batchDelete = async (criteria: FindOptionsWhere) => { + const batchSize = 1000 + + const qb = libraryItemRepository.createQueryBuilder().where(criteria) + const countSql = queryBuilderToRawSql(qb.select('COUNT(1)')) + const subQuery = queryBuilderToRawSql(qb.select('id').limit(batchSize)) + + const sql = ` + -- Set batch size + DO $$ + DECLARE + batch_size INT := ${batchSize}; + BEGIN + -- Loop through batches + FOR i IN 0..CEIL((${countSql}) * 1.0 / batch_size) - 1 LOOP + -- Delete batch + DELETE FROM omnivore.library_item + WHERE id = ANY( + ${subQuery} + ); + END LOOP; + END $$ + ` + + return authTrx(async (t) => t.query(sql)) +} diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts index f3d01e489..d7f9fa0d3 100644 --- a/packages/api/src/services/user.ts +++ b/packages/api/src/services/user.ts @@ -1,6 +1,6 @@ import { DeepPartial, FindOptionsWhere, In } from 'typeorm' import { StatusType, User } from '../entity/user' -import { authTrx } from '../repository' +import { authTrx, getRepository, queryBuilderToRawSql } from '../repository' import { userRepository } from '../repository/user' import { SetClaimsRole } from '../utils/dictionary' @@ -47,3 +47,49 @@ export const createUsers = async (users: DeepPartial[]) => { SetClaimsRole.ADMIN ) } + +export const batchDelete = async (criteria: FindOptionsWhere) => { + const userQb = getRepository(User).createQueryBuilder().where(criteria) + const userCountSql = queryBuilderToRawSql(userQb.select('COUNT(1)')) + const userSubQuery = queryBuilderToRawSql( + userQb.select('array_agg(id::UUID) into user_ids') + ) + + const batchSize = 1000 + const sql = ` + -- Set batch size + DO $$ + DECLARE + batch_size INT := ${batchSize}; + user_ids UUID[]; + BEGIN + -- Loop through batches of users + FOR i IN 0..CEIL((${userCountSql}) * 1.0 / batch_size) - 1 LOOP + -- GET batch of user ids + ${userSubQuery} LIMIT batch_size; + + -- Loop through batches of items + FOR j IN 0..CEIL((SELECT COUNT(1) FROM omnivore.library_item WHERE user_id = ANY(user_ids)) * 1.0 / batch_size) - 1 LOOP + -- Delete batch of items + DELETE FROM omnivore.library_item + WHERE id = ANY( + SELECT id + FROM omnivore.library_item + WHERE user_id = ANY(user_ids) + LIMIT batch_size + ); + END LOOP; + + -- Delete the batch of users + DELETE FROM omnivore.user WHERE id = ANY(user_ids); + END LOOP; + END $$ + ` + + return authTrx( + async (t) => t.query(sql), + undefined, + undefined, + SetClaimsRole.ADMIN + ) +} diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index b6e781597..090e31208 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -2285,4 +2285,58 @@ describe('Article API', () => { expect(item?.labels?.map((l) => l.name)).to.eql(['Favorites']) }) }) + + describe('EmptyTrash API', () => { + const emptyTrashQuery = () => ` + mutation { + emptyTrash { + ... on EmptyTrashSuccess { + success + } + ... on EmptyTrashError { + errorCodes + } + } + }` + + let items: LibraryItem[] = [] + + before(async () => { + // Create some test items + for (let i = 0; i < 5; i++) { + const itemToSave: DeepPartial = { + user, + title: 'test item', + readableContent: '

test

', + slug: '', + originalUrl: `https://blog.omnivore.app/p/empty-trash-${i}`, + deletedAt: new Date(), + state: LibraryItemState.Deleted, + } + const item = await createLibraryItem(itemToSave, user.id) + items.push(item) + } + }) + + after(async () => { + // Delete all items + await deleteLibraryItemsByUserId(user.id) + }) + + it('empties the trash', async () => { + let response = await graphqlRequest( + searchQuery('in:trash'), + authToken + ).expect(200) + expect(response.body.data.search.pageInfo.totalCount).to.eql(5) + + await graphqlRequest(emptyTrashQuery(), authToken).expect(200) + + response = await graphqlRequest( + searchQuery('in:trash'), + authToken + ).expect(200) + expect(response.body.data.search.pageInfo.totalCount).to.eql(0) + }) + }) })