diff --git a/packages/api/src/jobs/email/inbound_emails.ts b/packages/api/src/jobs/email/inbound_emails.ts index 5a047d51b..12528e884 100644 --- a/packages/api/src/jobs/email/inbound_emails.ts +++ b/packages/api/src/jobs/email/inbound_emails.ts @@ -23,10 +23,10 @@ import { enqueueSendEmail } from '../../utils/createTask' import { generateSlug, isUrl } from '../../utils/helpers' import { logger } from '../../utils/logger' import { - parseEmailAddress, - isProbablyArticle, - getTitleFromEmailSubject, generateUniqueUrl, + getTitleFromEmailSubject, + isProbablyArticle, + parseEmailAddress, } from '../../utils/parser' import { generateUploadFilePathName, diff --git a/packages/api/src/jobs/rss/refreshAllFeeds.ts b/packages/api/src/jobs/rss/refreshAllFeeds.ts index 29ef6f894..0f2cd8295 100644 --- a/packages/api/src/jobs/rss/refreshAllFeeds.ts +++ b/packages/api/src/jobs/rss/refreshAllFeeds.ts @@ -40,12 +40,11 @@ export const refreshAllFeeds = async (db: DataSource): Promise => { FROM omnivore.subscriptions s INNER JOIN - omnivore.user u ON u.id = s.user_id + omnivore.user u ON u.id = s.user_id AND u.status = $4 WHERE s.type = $1 AND s.status = $2 AND (s.scheduled_at <= NOW() OR s.scheduled_at IS NULL) - AND u.status = $4 GROUP BY url `, diff --git a/packages/api/src/routers/svc/user.ts b/packages/api/src/routers/svc/user.ts index ec6857ff0..1286a5cfa 100644 --- a/packages/api/src/routers/svc/user.ts +++ b/packages/api/src/routers/svc/user.ts @@ -37,24 +37,24 @@ export function userServiceRouter() { router.post('/prune', cors(corsConfig), async (req, res) => { logger.info('prune soft deleted users') - const { message: msgStr, expired } = readPushSubscription(req) + // const { message: msgStr, expired } = readPushSubscription(req) - if (!msgStr) { - return res.status(200).send('Bad Request') - } + // if (!msgStr) { + // return res.status(200).send('Bad Request') + // } - if (expired) { - logger.info('discarding expired message') - return res.status(200).send('Expired') - } + // if (expired) { + // logger.info('discarding expired message') + // return res.status(200).send('Expired') + // } - const cleanupMessage = getCleanupMessage(msgStr) - const subTime = cleanupMessage.subDays * 1000 * 60 * 60 * 24 // convert days to milliseconds + // const cleanupMessage = getCleanupMessage(msgStr) + // const subTime = cleanupMessage.subDays * 1000 * 60 * 60 * 24 // convert days to milliseconds try { const result = await batchDelete({ status: StatusType.Deleted, - updatedAt: LessThan(new Date(Date.now() - subTime)), // subDays ago + updatedAt: LessThan(new Date(Date.now() - 0)), // subDays ago }) logger.info('prune result', result) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 9a945a3a4..07628b4a6 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -1039,7 +1039,10 @@ export const updateLibraryItemReadingProgress = async ( } const updatedItem = result[0][0] - await pubsub.entityUpdated(EntityType.ITEM, updatedItem, userId) + if (updatedItem.readingProgressBottomPercent === 100) { + // mark item as read + await pubsub.entityUpdated(EntityType.ITEM, updatedItem, userId) + } return updatedItem } @@ -1332,6 +1335,7 @@ export const batchUpdateLibraryItems = async ( await authTrx( async (tx) => { const libraryItemIds = await getLibraryItemIds(userId, tx, true) + await tx.query(`SET lock_timeout = 10000; -- 10 seconds`) await tx.getRepository(LibraryItem).update(libraryItemIds, values) }, { diff --git a/packages/api/src/services/newsletters.ts b/packages/api/src/services/newsletters.ts index eb30ac267..281249057 100644 --- a/packages/api/src/services/newsletters.ts +++ b/packages/api/src/services/newsletters.ts @@ -1,5 +1,6 @@ import { nanoid } from 'nanoid' import { NewsletterEmail } from '../entity/newsletter_email' +import { StatusType } from '../entity/user' import { env } from '../env' import { CreateNewsletterEmailErrorCode, @@ -91,7 +92,12 @@ export const findNewsletterEmailByAddress = async ( const address = parsedAddress(emailAddress) return getRepository(NewsletterEmail) .createQueryBuilder('newsletter_email') - .innerJoinAndSelect('newsletter_email.user', 'user') + .innerJoinAndSelect( + 'newsletter_email.user', + 'user', + 'user.status = :status', + { status: StatusType.Active } + ) .where('LOWER(address) = :address', { address: address.toLowerCase() }) .getOne() } diff --git a/packages/api/src/services/rules.ts b/packages/api/src/services/rules.ts index c0c394fb4..5e380b2ab 100644 --- a/packages/api/src/services/rules.ts +++ b/packages/api/src/services/rules.ts @@ -1,5 +1,6 @@ import { ArrayContains, ILike, IsNull, Not } from 'typeorm' import { Rule, RuleAction, RuleEventType } from '../entity/rule' +import { StatusType } from '../entity/user' import { authTrx, getRepository } from '../repository' export const createRule = async ( @@ -62,7 +63,7 @@ export const findEnabledRules = async ( eventType: RuleEventType ) => { return getRepository(Rule).findBy({ - user: { id: userId }, + user: { id: userId, status: StatusType.Active }, enabled: true, eventTypes: ArrayContains([eventType]), failedAt: IsNull(), // only rules that have not failed diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts index c5dc336b9..32f3edf12 100644 --- a/packages/api/src/services/user.ts +++ b/packages/api/src/services/user.ts @@ -81,38 +81,20 @@ export const createUsers = async (users: DeepPartial[]) => { export const batchDelete = async (criteria: FindOptionsWhere) => { const userQb = getRepository(User).createQueryBuilder().where(criteria) - const userCountSql = queryBuilderToRawSql(userQb.select('COUNT(1)')) - const userSubQuery = queryBuilderToRawSql( - userQb.select('array_agg(id::UUID) into user_ids') - ) + const batchSize = 100 + const userSubQuery = queryBuilderToRawSql(userQb.select('id').take(batchSize)) - const batchSize = 1000 const sql = ` - -- Set batch size DO $$ - DECLARE - batch_size INT := ${batchSize}; - user_ids UUID[]; BEGIN - -- Loop through batches of users - FOR i IN 0..CEIL((${userCountSql}) * 1.0 / batch_size) - 1 LOOP - -- GET batch of user ids - ${userSubQuery} LIMIT batch_size; + LOOP + DELETE FROM omnivore.user + WHERE id IN (${userSubQuery}); - -- Loop through batches of items - FOR j IN 0..CEIL((SELECT COUNT(1) FROM omnivore.library_item WHERE user_id = ANY(user_ids)) * 1.0 / batch_size) - 1 LOOP - -- Delete batch of items - DELETE FROM omnivore.library_item - WHERE id = ANY( - SELECT id - FROM omnivore.library_item - WHERE user_id = ANY(user_ids) - LIMIT batch_size - ); - END LOOP; + EXIT WHEN NOT FOUND; - -- Delete the batch of users - DELETE FROM omnivore.user WHERE id = ANY(user_ids); + -- Avoid overwhelming the server + PERFORM pg_sleep(0.1); END LOOP; END $$ ` diff --git a/packages/api/test/resolvers/newsletters.test.ts b/packages/api/test/resolvers/newsletters.test.ts index 6f48e3ea6..8fcc464f0 100644 --- a/packages/api/test/resolvers/newsletters.test.ts +++ b/packages/api/test/resolvers/newsletters.test.ts @@ -30,7 +30,7 @@ describe('Newsletters API', () => { .post('/local/debug/fake-user-login') .send({ fakeEmail: user.email }) - authToken = res.body.authToken + authToken = res.body.authToken as string }) after(async () => { @@ -65,14 +65,8 @@ describe('Newsletters API', () => { before(async () => { // create test newsletter emails - const newsletterEmail1 = await createNewsletterEmail( - user.id, - 'Test_email_address_1@omnivore.app' - ) - const newsletterEmail2 = await createNewsletterEmail( - user.id, - 'Test_email_address_2@omnivore.app' - ) + const newsletterEmail1 = await createNewsletterEmail(user.id) + const newsletterEmail2 = await createNewsletterEmail(user.id) newsletterEmails = [newsletterEmail1, newsletterEmail2] // create testing subscriptions @@ -89,7 +83,9 @@ describe('Newsletters API', () => { it('responds with newsletter emails sort by created_at desc', async () => { const response = await graphqlRequest(query, authToken).expect(200) expect( + // eslint-disable-next-line @typescript-eslint/no-unsafe-call response.body.data.newsletterEmails.newsletterEmails.map((e: any) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-return return { ...e, createdAt: @@ -124,10 +120,7 @@ describe('Newsletters API', () => { before(async () => { // create test newsletter emails - newsletterEmail = await createNewsletterEmail( - user.id, - 'Test_email_address_1@omnivore.app' - ) + newsletterEmail = await createNewsletterEmail(user.id) // create unsubscribed subscriptions await createSubscription( @@ -190,7 +183,7 @@ describe('Newsletters API', () => { const response = await graphqlRequest(query, authToken, { input: { folder, - } + }, }).expect(200) const newsletterEmail = await findNewsletterEmailById( response.body.data.createNewsletterEmail.newsletterEmail.id @@ -239,10 +232,7 @@ describe('Newsletters API', () => { context('when newsletter email exists', () => { before(async () => { // create test newsletter emails - const newsletterEmail = await createNewsletterEmail( - user.id, - 'Test_email_address_1@omnivore.app' - ) + const newsletterEmail = await createNewsletterEmail(user.id) newsletterEmailId = newsletterEmail.id }) @@ -254,7 +244,7 @@ describe('Newsletters API', () => { it('responds with status code 200', async () => { const response = await graphqlRequest(query, authToken).expect(200) const newsletterEmail = await findNewsletterEmailByAddress( - response.body.data.deleteNewsletterEmail.newsletterEmail.id + response.body.data.deleteNewsletterEmail.newsletterEmail.address ) expect(newsletterEmail).to.be.null }) diff --git a/packages/db/migrations/0187.do.allow_admin_to_delete_filters.sql b/packages/db/migrations/0187.do.allow_admin_to_delete_filters.sql new file mode 100755 index 000000000..b60d73dd1 --- /dev/null +++ b/packages/db/migrations/0187.do.allow_admin_to_delete_filters.sql @@ -0,0 +1,14 @@ +-- Type: DO +-- Name: allow_admin_to_delete_filters +-- Description: Add permissions to delete data from filters table to the omnivore_admin role + +BEGIN; + +GRANT SELECT, INSERT, UPDATE, DELETE ON omnivore.filters TO omnivore_admin; + +CREATE POLICY filters_admin_policy on omnivore.filters + FOR ALL + TO omnivore_admin + USING (true); + +COMMIT; diff --git a/packages/db/migrations/0187.undo.allow_admin_to_delete_filters.sql b/packages/db/migrations/0187.undo.allow_admin_to_delete_filters.sql new file mode 100755 index 000000000..426665792 --- /dev/null +++ b/packages/db/migrations/0187.undo.allow_admin_to_delete_filters.sql @@ -0,0 +1,11 @@ +-- Type: UNDO +-- Name: allow_admin_to_delete_filters +-- Description: Add permissions to delete data from filters table to the omnivore_admin role + +BEGIN; + +DROP POLICY filters_admin_policy on omnivore.filters; + +REVOKE SELECT, INSERT, UPDATE, DELETE ON omnivore.filters FROM omnivore_admin; + +COMMIT;