Merge pull request #3300 from omnivore-app/fix/batch-prune

fix/batch prune

packages/api/src/generated/graphql.ts

@@ -727,6 +727,22 @@ export type DeviceTokensSuccess = {
   deviceTokens: Array<DeviceToken>;
 };
 
+export type EmptyTrashError = {
+  __typename?: 'EmptyTrashError';
+  errorCodes: Array<EmptyTrashErrorCode>;
+};
+
+export enum EmptyTrashErrorCode {
+  Unauthorized = 'UNAUTHORIZED'
+}
+
+export type EmptyTrashResult = EmptyTrashError | EmptyTrashSuccess;
+
+export type EmptyTrashSuccess = {
+  __typename?: 'EmptyTrashSuccess';
+  success?: Maybe<Scalars['Boolean']>;
+};
+
 export type Feature = {
   __typename?: 'Feature';
   createdAt: Scalars['Date'];
@@ -1352,6 +1368,7 @@ export type Mutation = {
   deleteNewsletterEmail: DeleteNewsletterEmailResult;
   deleteRule: DeleteRuleResult;
   deleteWebhook: DeleteWebhookResult;
+  emptyTrash: EmptyTrashResult;
   fetchContent: FetchContentResult;
   generateApiKey: GenerateApiKeyResult;
   googleLogin: LoginResult;
@@ -3663,6 +3680,10 @@ export type ResolversTypes = {
   DeviceTokensErrorCode: DeviceTokensErrorCode;
   DeviceTokensResult: ResolversTypes['DeviceTokensError'] | ResolversTypes['DeviceTokensSuccess'];
   DeviceTokensSuccess: ResolverTypeWrapper<DeviceTokensSuccess>;
+  EmptyTrashError: ResolverTypeWrapper<EmptyTrashError>;
+  EmptyTrashErrorCode: EmptyTrashErrorCode;
+  EmptyTrashResult: ResolversTypes['EmptyTrashError'] | ResolversTypes['EmptyTrashSuccess'];
+  EmptyTrashSuccess: ResolverTypeWrapper<EmptyTrashSuccess>;
   Feature: ResolverTypeWrapper<Feature>;
   Feed: ResolverTypeWrapper<Feed>;
   FeedArticle: ResolverTypeWrapper<FeedArticle>;
@@ -4169,6 +4190,9 @@ export type ResolversParentTypes = {
   DeviceTokensError: DeviceTokensError;
   DeviceTokensResult: ResolversParentTypes['DeviceTokensError'] | ResolversParentTypes['DeviceTokensSuccess'];
   DeviceTokensSuccess: DeviceTokensSuccess;
+  EmptyTrashError: EmptyTrashError;
+  EmptyTrashResult: ResolversParentTypes['EmptyTrashError'] | ResolversParentTypes['EmptyTrashSuccess'];
+  EmptyTrashSuccess: EmptyTrashSuccess;
   Feature: Feature;
   Feed: Feed;
   FeedArticle: FeedArticle;
@@ -4975,6 +4999,20 @@ export type DeviceTokensSuccessResolvers<ContextType = ResolverContext, ParentTy
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type EmptyTrashErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['EmptyTrashError'] = ResolversParentTypes['EmptyTrashError']> = {
+  errorCodes?: Resolver<Array<ResolversTypes['EmptyTrashErrorCode']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type EmptyTrashResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['EmptyTrashResult'] = ResolversParentTypes['EmptyTrashResult']> = {
+  __resolveType: TypeResolveFn<'EmptyTrashError' | 'EmptyTrashSuccess', ParentType, ContextType>;
+};
+
+export type EmptyTrashSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['EmptyTrashSuccess'] = ResolversParentTypes['EmptyTrashSuccess']> = {
+  success?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type FeatureResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Feature'] = ResolversParentTypes['Feature']> = {
   createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
   expiresAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
@@ -5460,6 +5498,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
   deleteNewsletterEmail?: Resolver<ResolversTypes['DeleteNewsletterEmailResult'], ParentType, ContextType, RequireFields<MutationDeleteNewsletterEmailArgs, 'newsletterEmailId'>>;
   deleteRule?: Resolver<ResolversTypes['DeleteRuleResult'], ParentType, ContextType, RequireFields<MutationDeleteRuleArgs, 'id'>>;
   deleteWebhook?: Resolver<ResolversTypes['DeleteWebhookResult'], ParentType, ContextType, RequireFields<MutationDeleteWebhookArgs, 'id'>>;
+  emptyTrash?: Resolver<ResolversTypes['EmptyTrashResult'], ParentType, ContextType>;
   fetchContent?: Resolver<ResolversTypes['FetchContentResult'], ParentType, ContextType, RequireFields<MutationFetchContentArgs, 'id'>>;
   generateApiKey?: Resolver<ResolversTypes['GenerateApiKeyResult'], ParentType, ContextType, RequireFields<MutationGenerateApiKeyArgs, 'input'>>;
   googleLogin?: Resolver<ResolversTypes['LoginResult'], ParentType, ContextType, RequireFields<MutationGoogleLoginArgs, 'input'>>;
@@ -6647,6 +6686,9 @@ export type Resolvers<ContextType = ResolverContext> = {
   DeviceTokensError?: DeviceTokensErrorResolvers<ContextType>;
   DeviceTokensResult?: DeviceTokensResultResolvers<ContextType>;
   DeviceTokensSuccess?: DeviceTokensSuccessResolvers<ContextType>;
+  EmptyTrashError?: EmptyTrashErrorResolvers<ContextType>;
+  EmptyTrashResult?: EmptyTrashResultResolvers<ContextType>;
+  EmptyTrashSuccess?: EmptyTrashSuccessResolvers<ContextType>;
   Feature?: FeatureResolvers<ContextType>;
   Feed?: FeedResolvers<ContextType>;
   FeedArticle?: FeedArticleResolvers<ContextType>;

packages/api/src/generated/schema.graphql

@@ -644,6 +644,20 @@ type DeviceTokensSuccess {
   deviceTokens: [DeviceToken!]!
 }
 
+type EmptyTrashError {
+  errorCodes: [EmptyTrashErrorCode!]!
+}
+
+enum EmptyTrashErrorCode {
+  UNAUTHORIZED
+}
+
+union EmptyTrashResult = EmptyTrashError | EmptyTrashSuccess
+
+type EmptyTrashSuccess {
+  success: Boolean
+}
+
 type Feature {
   createdAt: Date!
   expiresAt: Date
@@ -1213,6 +1227,7 @@ type Mutation {
   deleteNewsletterEmail(newsletterEmailId: ID!): DeleteNewsletterEmailResult!
   deleteRule(id: ID!): DeleteRuleResult!
   deleteWebhook(id: ID!): DeleteWebhookResult!
+  emptyTrash: EmptyTrashResult!
   fetchContent(id: ID!): FetchContentResult!
   generateApiKey(input: GenerateApiKeyInput!): GenerateApiKeyResult!
   googleLogin(input: GoogleLoginInput!): LoginResult!

packages/api/src/resolvers/article/index.ts

@@ -21,6 +21,8 @@ import {
   CreateArticleError,
   CreateArticleErrorCode,
   CreateArticleSuccess,
+  EmptyTrashError,
+  EmptyTrashSuccess,
   FetchContentError,
   FetchContentErrorCode,
   FetchContentSuccess,
@@ -71,6 +73,7 @@ import {
   findOrCreateLabels,
 } from '../../services/labels'
 import {
+  batchDelete,
   batchUpdateLibraryItems,
   createLibraryItem,
   findLibraryItemById,
@@ -1034,6 +1037,27 @@ export const fetchContentResolver = authorized<
   }
 })
 
+export const emptyTrashResolver = authorized<
+  EmptyTrashSuccess,
+  EmptyTrashError
+>(async (_, __, { uid }) => {
+  analytics.track({
+    userId: uid,
+    event: 'empty_trash',
+  })
+
+  await batchDelete({
+    state: LibraryItemState.Deleted,
+    user: {
+      id: uid,
+    },
+  })
+
+  return {
+    success: true,
+  }
+})
+
 const getUpdateReason = (libraryItem: LibraryItem, since: Date) => {
   if (libraryItem.deletedAt) {
     return UpdateReason.Deleted

packages/api/src/resolvers/function_resolvers.ts

@@ -38,6 +38,7 @@ import {
   generateDownloadSignedUrl,
   generateUploadFilePathName,
 } from '../utils/uploads'
+import { emptyTrashResolver } from './article'
 import { optInFeatureResolver } from './features'
 import { uploadImportFileResolver } from './importers/uploadImportFileResolver'
 import {
@@ -231,6 +232,7 @@ export const functionResolvers = {
     updateEmail: updateEmailResolver,
     moveToFolder: moveToFolderResolver,
     updateNewsletterEmail: updateNewsletterEmailResolver,
+    emptyTrash: emptyTrashResolver,
   },
   Query: {
     me: getMeUserResolver,
@@ -578,4 +580,5 @@ export const functionResolvers = {
   ...resultResolveTypeResolver('ScanFeeds'),
   ...resultResolveTypeResolver('MoveToFolder'),
   ...resultResolveTypeResolver('UpdateNewsletterEmail'),
+  ...resultResolveTypeResolver('EmptyTrash'),
 }

packages/api/src/routers/svc/user.ts

@@ -5,7 +5,7 @@ import express from 'express'
 import { LessThan } from 'typeorm'
 import { StatusType } from '../../entity/user'
 import { readPushSubscription } from '../../pubsub'
-import { deleteUsers } from '../../services/user'
+import { batchDelete } from '../../services/user'
 import { corsConfig } from '../../utils/corsConfig'
 import { logger } from '../../utils/logger'
 
@@ -52,7 +52,7 @@ export function userServiceRouter() {
     const subTime = cleanupMessage.subDays * 1000 * 60 * 60 * 24 // convert days to milliseconds
 
     try {
-      const result = await deleteUsers({
+      const result = await batchDelete({
         status: StatusType.Deleted,
         updatedAt: LessThan(new Date(Date.now() - subTime)), // subDays ago
       })

packages/api/src/schema.ts

@@ -2761,6 +2761,20 @@ const schema = gql`
     BAD_REQUEST
   }
 
+  union EmptyTrashResult = EmptyTrashSuccess | EmptyTrashError
+
+  type EmptyTrashSuccess {
+    success: Boolean
+  }
+
+  type EmptyTrashError {
+    errorCodes: [EmptyTrashErrorCode!]!
+  }
+
+  enum EmptyTrashErrorCode {
+    UNAUTHORIZED
+  }
+
   # Mutations
   type Mutation {
     googleLogin(input: GoogleLoginInput!): LoginResult!
@@ -2872,6 +2886,7 @@ const schema = gql`
     updateNewsletterEmail(
       input: UpdateNewsletterEmailInput!
     ): UpdateNewsletterEmailResult!
+    emptyTrash: EmptyTrashResult!
   }
 
   # FIXME: remove sort from feedArticles after all cached tabs are closed

packages/api/src/services/library_item.ts

@@ -1059,3 +1059,30 @@ export const deleteLibraryItemsByAdmin = async (
     'admin'
   )
 }
+
+export const batchDelete = async (criteria: FindOptionsWhere<LibraryItem>) => {
+  const batchSize = 1000
+
+  const qb = libraryItemRepository.createQueryBuilder().where(criteria)
+  const countSql = queryBuilderToRawSql(qb.select('COUNT(1)'))
+  const subQuery = queryBuilderToRawSql(qb.select('id').limit(batchSize))
+
+  const sql = `
+  -- Set batch size
+  DO $$
+  DECLARE 
+      batch_size INT := ${batchSize};
+  BEGIN
+      -- Loop through batches
+      FOR i IN 0..CEIL((${countSql}) * 1.0 / batch_size) - 1 LOOP
+          -- Delete batch
+          DELETE FROM omnivore.library_item
+          WHERE id = ANY(
+            ${subQuery}
+          );
+      END LOOP;
+  END $$
+  `
+
+  return authTrx(async (t) => t.query(sql))
+}

packages/api/src/services/user.ts

@@ -1,6 +1,6 @@
 import { DeepPartial, FindOptionsWhere, In } from 'typeorm'
 import { StatusType, User } from '../entity/user'
-import { authTrx } from '../repository'
+import { authTrx, getRepository, queryBuilderToRawSql } from '../repository'
 import { userRepository } from '../repository/user'
 import { SetClaimsRole } from '../utils/dictionary'
 
@@ -47,3 +47,49 @@ export const createUsers = async (users: DeepPartial<User>[]) => {
     SetClaimsRole.ADMIN
   )
 }
+
+export const batchDelete = async (criteria: FindOptionsWhere<User>) => {
+  const userQb = getRepository(User).createQueryBuilder().where(criteria)
+  const userCountSql = queryBuilderToRawSql(userQb.select('COUNT(1)'))
+  const userSubQuery = queryBuilderToRawSql(
+    userQb.select('array_agg(id::UUID) into user_ids')
+  )
+
+  const batchSize = 1000
+  const sql = `
+  -- Set batch size
+  DO $$
+  DECLARE 
+      batch_size INT := ${batchSize};
+      user_ids UUID[];
+  BEGIN
+      -- Loop through batches of users
+      FOR i IN 0..CEIL((${userCountSql}) * 1.0 / batch_size) - 1 LOOP
+          -- GET batch of user ids
+          ${userSubQuery} LIMIT batch_size;
+          
+          -- Loop through batches of items
+          FOR j IN 0..CEIL((SELECT COUNT(1) FROM omnivore.library_item WHERE user_id = ANY(user_ids)) * 1.0 / batch_size) - 1 LOOP
+              -- Delete batch of items
+              DELETE FROM omnivore.library_item
+              WHERE id = ANY(
+                SELECT id
+                FROM omnivore.library_item
+                WHERE user_id = ANY(user_ids)
+                LIMIT batch_size
+              );
+          END LOOP;
+
+          -- Delete the batch of users
+          DELETE FROM omnivore.user WHERE id = ANY(user_ids);
+      END LOOP;
+  END $$
+  `
+
+  return authTrx(
+    async (t) => t.query(sql),
+    undefined,
+    undefined,
+    SetClaimsRole.ADMIN
+  )
+}

packages/api/test/resolvers/article.test.ts

@@ -2285,4 +2285,58 @@ describe('Article API', () => {
       expect(item?.labels?.map((l) => l.name)).to.eql(['Favorites'])
     })
   })
+
+  describe('EmptyTrash API', () => {
+    const emptyTrashQuery = () => `
+      mutation {
+        emptyTrash {
+          ... on EmptyTrashSuccess {
+            success
+          }
+          ... on EmptyTrashError {
+            errorCodes
+          }
+        }
+      }`
+
+    let items: LibraryItem[] = []
+
+    before(async () => {
+      // Create some test items
+      for (let i = 0; i < 5; i++) {
+        const itemToSave: DeepPartial<LibraryItem> = {
+          user,
+          title: 'test item',
+          readableContent: '<p>test</p>',
+          slug: '',
+          originalUrl: `https://blog.omnivore.app/p/empty-trash-${i}`,
+          deletedAt: new Date(),
+          state: LibraryItemState.Deleted,
+        }
+        const item = await createLibraryItem(itemToSave, user.id)
+        items.push(item)
+      }
+    })
+
+    after(async () => {
+      // Delete all items
+      await deleteLibraryItemsByUserId(user.id)
+    })
+
+    it('empties the trash', async () => {
+      let response = await graphqlRequest(
+        searchQuery('in:trash'),
+        authToken
+      ).expect(200)
+      expect(response.body.data.search.pageInfo.totalCount).to.eql(5)
+
+      await graphqlRequest(emptyTrashQuery(), authToken).expect(200)
+
+      response = await graphqlRequest(
+        searchQuery('in:trash'),
+        authToken
+      ).expect(200)
+      expect(response.body.data.search.pageInfo.totalCount).to.eql(0)
+    })
+  })
 })