Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4475 from omnivore-app/jacksonh/disable-imports

Browse Source 
Disable the importer to give more resources to exports
This commit is contained in:
Jackson Harper
2024-11-01 23:30:49 +08:00
committed by GitHub
parent f16444c893 ee2b901fb6
commit cd0ac4ba30
5 changed files with 149 additions and 618 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
packages/api/src/resolvers/integrations/index.ts
View File

@ -218,48 +218,48 @@ export const importFromIntegrationResolver = authorized<
ImportFromIntegrationSuccess,
ImportFromIntegrationError,
MutationImportFromIntegrationArgs
>(async (_, { integrationId }, { claims: { uid }, log }) => {
const integration = await findIntegration({ id: integrationId }, uid)
if (!integration) {
return {
errorCodes: [ImportFromIntegrationErrorCode.Unauthorized],
}
}
const authToken = await createIntegrationToken({
uid: integration.user.id,
token: integration.token,
})
if (!authToken) {
return {
errorCodes: [ImportFromIntegrationErrorCode.BadRequest],
}
}
// create a task to import all the pages
const taskName = await enqueueImportFromIntegration(
integration.id,
integration.name,
integration.syncedAt?.getTime() || 0,
authToken,
integration.importItemState || ImportItemState.Unarchived
)
log.info('task created', taskName)
// // update task name in integration
// await updateIntegration(integration.id, { taskName }, uid)
analytics.capture({
distinctId: uid,
event: 'integration_import',
properties: {
integrationId,
},
})
>((_, { integrationId }, { claims: { uid }, log }) => {
// const integration = await findIntegration({ id: integrationId }, uid)
// if (!integration) {
return {
success: true,
errorCodes: [ImportFromIntegrationErrorCode.Unauthorized],
}
// }
// const authToken = await createIntegrationToken({
// uid: integration.user.id,
// token: integration.token,
// })
// if (!authToken) {
// return {
// errorCodes: [ImportFromIntegrationErrorCode.BadRequest],
// }
// }
// // create a task to import all the pages
// const taskName = await enqueueImportFromIntegration(
// integration.id,
// integration.name,
// integration.syncedAt?.getTime() || 0,
// authToken,
// integration.importItemState || ImportItemState.Unarchived
// )
// log.info('task created', taskName)
// // // update task name in integration
// // await updateIntegration(integration.id, { taskName }, uid)
// analytics.capture({
// distinctId: uid,
// event: 'integration_import',
// properties: {
// integrationId,
// },
// })
// return {
// success: true,
// }
})
export const exportToIntegrationResolver = authorized<

477
packages/api/test/resolvers/integrations.test.ts
View File

@ -1,477 +0,0 @@
import chai, { expect } from 'chai'
import 'mocha'
import nock from 'nock'
import sinonChai from 'sinon-chai'
import { Integration } from '../../src/entity/integration'
import { User } from '../../src/entity/user'
import { SetIntegrationErrorCode } from '../../src/generated/graphql'
import {
deleteIntegrations,
findIntegration,
saveIntegration,
updateIntegration,
} from '../../src/services/integrations'
import { deleteUser } from '../../src/services/user'
import { createTestUser } from '../db'
import { generateFakeUuid, graphqlRequest, request } from '../util'
chai.use(sinonChai)
describe('Integrations resolvers', () => {
const READWISE_API_URL = 'https://readwise.io/api/v2'
let loginUser: User
let authToken: string
before(async () => {
// create test user and login
loginUser = await createTestUser('loginUser')
const res = await request
.post('/local/debug/fake-user-login')
.send({ fakeEmail: loginUser.email })
authToken = res.body.authToken as string
})
after(async () => {
await deleteUser(loginUser.id)
})
describe('setIntegration API', () => {
const validToken = 'valid-token'
const query = `
mutation SetIntegration($input: SetIntegrationInput!) {
setIntegration(input: $input) {
... on SetIntegrationSuccess {
integration {
id
enabled
}
}
... on SetIntegrationError {
errorCodes
}
}
}
`
let integrationId: string
let token: string
let integrationName: string
let enabled: boolean
let scope: nock.Scope
// mock Readwise Auth API
before(() => {
scope = nock(READWISE_API_URL, {
reqheaders: { Authorization: `Token ${validToken}` },
})
.get('/auth')
.reply(204)
.persist()
integrationName = 'READWISE'
enabled = true
token = 'test token'
})
after(() => {
scope.persist(false)
})
context('when id is not in the request', () => {
before(() => {
integrationId = ''
})
context('when token is invalid', () => {
before(() => {
token = 'invalid token'
nock(READWISE_API_URL, {
reqheaders: { Authorization: `Token ${token}` },
})
.get('/auth')
.reply(401)
})
it('returns InvalidToken error code', async () => {
const res = await graphqlRequest(query, authToken, {
input: {
id: integrationId,
name: integrationName,
token,
enabled,
},
})
expect(res.body.data.setIntegration.errorCodes).to.eql([
SetIntegrationErrorCode.InvalidToken,
])
})
})
context('when token is valid', () => {
before(() => {
token = validToken
})
afterEach(async () => {
await deleteIntegrations(loginUser.id, {
user: { id: loginUser.id },
name: integrationName,
})
})
it('creates new integration', async () => {
const res = await graphqlRequest(query, authToken, {
input: {
id: integrationId,
name: integrationName,
token,
enabled,
},
})
expect(res.body.data.setIntegration.integration.enabled).to.be.true
})
})
})
context('when id is in the request', () => {
let existingIntegration: Integration
context('when integration does not exist', () => {
before(() => {
integrationId = generateFakeUuid()
})
it('returns NotFound error code', async () => {
const res = await graphqlRequest(query, authToken, {
input: { id: integrationId, name: integrationName, enabled, token },
})
expect(res.body.data.setIntegration.errorCodes).to.eql([
SetIntegrationErrorCode.NotFound,
])
})
})
context('when integration exists', () => {
context('when integration does not belong to the user', () => {
let otherUser: User
before(async () => {
otherUser = await createTestUser('otherUser')
existingIntegration = await saveIntegration(
{
user: { id: otherUser.id },
name: 'READWISE',
token: 'fakeToken',
enabled,
},
otherUser.id
)
integrationId = existingIntegration.id
})
after(async () => {
await deleteUser(otherUser.id)
await deleteIntegrations(otherUser.id, [existingIntegration.id])
})
it('returns Unauthorized error code', async () => {
const res = await graphqlRequest(query, authToken, {
input: {
id: integrationId,
name: integrationName,
enabled,
token,
},
})
expect(res.body.data.setIntegration.errorCodes).to.eql([
SetIntegrationErrorCode.NotFound,
])
})
})
context('when integration belongs to the user', () => {
before(async () => {
existingIntegration = await saveIntegration(
{
user: { id: loginUser.id },
name: 'READWISE',
token: 'fakeToken',
enabled,
},
loginUser.id
)
integrationId = existingIntegration.id
})
after(async () => {
await deleteIntegrations(loginUser.id, [existingIntegration.id])
})
context('when enable is false', () => {
before(() => {
enabled = false
})
afterEach(async () => {
await updateIntegration(
existingIntegration.id,
{
taskName: 'some task name',
enabled: true,
},
loginUser.id
)
})
it('disables integration', async () => {
const res = await graphqlRequest(query, authToken, {
input: {
id: integrationId,
name: integrationName,
token,
enabled,
},
})
expect(res.body.data.setIntegration.integration.enabled).to.be
.false
})
})
context('when enable is true', () => {
before(() => {
enabled = true
})
afterEach(async () => {
await updateIntegration(
existingIntegration.id,
{
taskName: null,
enabled: false,
},
loginUser.id
)
})
it('enables integration', async () => {
const res = await graphqlRequest(query, authToken, {
input: {
id: integrationId,
name: integrationName,
token,
enabled,
},
})
expect(res.body.data.setIntegration.integration.enabled).to.be
.true
})
})
})
})
})
})
describe('integrations API', () => {
const query = `
query {
integrations {
... on IntegrationsSuccess {
integrations {
id
type
enabled
}
}
}
}
`
let existingIntegration: Integration
before(async () => {
existingIntegration = await saveIntegration(
{
user: { id: loginUser.id },
name: 'READWISE',
token: 'fakeToken',
},
loginUser.id
)
})
after(async () => {
await deleteIntegrations(loginUser.id, [existingIntegration.id])
})
it('returns all integrations', async () => {
const res = await graphqlRequest(query, authToken)
expect(res.body.data.integrations.integrations).to.have.length(1)
expect(res.body.data.integrations.integrations[0].id).to.equal(
existingIntegration.id
)
expect(res.body.data.integrations.integrations[0].type).to.equal(
existingIntegration.type
)
expect(res.body.data.integrations.integrations[0].enabled).to.equal(
existingIntegration.enabled
)
})
})
describe('deleteIntegration API', () => {
const query = (id: string) => `
mutation {
deleteIntegration(id: "${id}") {
... on DeleteIntegrationSuccess {
integration {
id
}
}
... on DeleteIntegrationError {
errorCodes
}
}
}
`
context('when integration exists', () => {
let existingIntegration: Integration
beforeEach(async () => {
existingIntegration = await saveIntegration(
{
user: { id: loginUser.id },
name: 'READWISE',
token: 'fakeToken',
taskName: 'some task name',
},
loginUser.id
)
})
it('deletes the integration and cloud task', async () => {
const res = await graphqlRequest(
query(existingIntegration.id),
authToken
)
const integration = await findIntegration(
{
id: existingIntegration.id,
},
loginUser.id
)
expect(res.body.data.deleteIntegration.integration).to.be.an('object')
expect(res.body.data.deleteIntegration.integration.id).to.eql(
existingIntegration.id
)
expect(integration).to.be.null
})
})
})
describe('importFromIntegration API', () => {
const query = (integrationId: string) => `
mutation {
importFromIntegration(integrationId: "${integrationId}") {
... on ImportFromIntegrationSuccess {
success
}
... on ImportFromIntegrationError {
errorCodes
}
}
}
`
let existingIntegration: Integration
context('when integration exists', () => {
before(async () => {
existingIntegration = await saveIntegration(
{
user: { id: loginUser.id },
name: 'POCKET',
token: 'fakeToken',
},
loginUser.id
)
})
after(async () => {
await deleteIntegrations(loginUser.id, [existingIntegration.id])
})
it('returns success and starts cloud task', async () => {
const res = await graphqlRequest(
query(existingIntegration.id),
authToken
).expect(200)
expect(res.body.data.importFromIntegration.success).to.be.true
})
})
context('when integration does not exist', () => {
it('returns error', async () => {
const invalidIntegrationId = generateFakeUuid()
const res = await graphqlRequest(
query(invalidIntegrationId),
authToken
).expect(200)
expect(res.body.data.importFromIntegration.errorCodes).to.eql([
'UNAUTHORIZED',
])
})
})
})
describe('integration API', () => {
const query = `
query Integration ($name: String!) {
integration(name: $name) {
... on IntegrationSuccess {
integration {
id
type
enabled
}
}
... on IntegrationError {
errorCodes
}
}
}
`
let existingIntegration: Integration
before(async () => {
existingIntegration = await saveIntegration(
{
user: { id: loginUser.id },
name: 'READWISE',
token: 'fakeToken',
},
loginUser.id
)
})
after(async () => {
await deleteIntegrations(loginUser.id, [existingIntegration.id])
})
it('returns the integration', async () => {
const res = await graphqlRequest(query, authToken, {
name: existingIntegration.name,
})
expect(res.body.data.integration.integration.id).to.equal(
existingIntegration.id
)
expect(res.body.data.integration.integration.type).to.equal(
existingIntegration.type
)
expect(res.body.data.integration.integration.enabled).to.equal(
existingIntegration.enabled
)
})
})
})

103
packages/web/components/templates/AddLinkModal.tsx
View File

@ -62,17 +62,17 @@ export function AddLinkModal(props: AddLinkModalProps): JSX.Element {
}}
>
<VStack distribution="start" css={{ gap: '20px' }}>
<TabBar
{/* <TabBar
selectedTab={selectedTab}
setSelectedTab={setSelectedTab}
onOpenChange={props.onOpenChange}
/>
/> */}
<Box css={{ width: '100%' }}>
{selectedTab == 'link' && <AddLinkTab {...props} />}
{selectedTab == 'feed' && <AddFeedTab {...props} />}
{/* {selectedTab == 'feed' && <AddFeedTab {...props} />}
{selectedTab == 'opml' && <UploadOPMLTab />}
{selectedTab == 'pdf' && <UploadPDFTab />}
{selectedTab == 'import' && <UploadImportTab {...props} />}
{selectedTab == 'import' && <UploadImportTab {...props} />} */}
</Box>
</VStack>
</ModalContent>
@ -531,52 +531,52 @@ const UploadPad = (props: UploadPadProps): JSX.Element => {
const allFiles = [...uploadFiles, ...addedFiles]
setUploadFiles(allFiles)
; (async () => {
for (const file of addedFiles) {
try {
const uploadInfo = await uploadSignedUrlForFile(file)
if (!uploadInfo.uploadSignedUrl) {
const message = uploadInfo.message || 'No upload URL available'
showErrorToast(message, { duration: 10000 })
file.status = 'error'
setUploadFiles([...allFiles])
return
}
const uploadResult = await axios.request({
method: 'PUT',
url: uploadInfo.uploadSignedUrl,
data: file.file,
withCredentials: false,
headers: {
'Content-Type': file.file.type,
},
onUploadProgress: (p) => {
if (!p.total) {
console.warn('No total available for upload progress')
return
}
const progress = (p.loaded / p.total) * 100
file.progress = progress
setUploadFiles([...allFiles])
},
})
file.progress = 100
file.status = 'success'
file.openUrl = uploadInfo.requestId
? `/article/sr/${uploadInfo.requestId}`
: undefined
file.message = uploadInfo.message
setUploadFiles([...allFiles])
} catch (error) {
;(async () => {
for (const file of addedFiles) {
try {
const uploadInfo = await uploadSignedUrlForFile(file)
if (!uploadInfo.uploadSignedUrl) {
const message = uploadInfo.message || 'No upload URL available'
showErrorToast(message, { duration: 10000 })
file.status = 'error'
setUploadFiles([...allFiles])
return
}
const uploadResult = await axios.request({
method: 'PUT',
url: uploadInfo.uploadSignedUrl,
data: file.file,
withCredentials: false,
headers: {
'Content-Type': file.file.type,
},
onUploadProgress: (p) => {
if (!p.total) {
console.warn('No total available for upload progress')
return
}
const progress = (p.loaded / p.total) * 100
file.progress = progress
setUploadFiles([...allFiles])
},
})
file.progress = 100
file.status = 'success'
file.openUrl = uploadInfo.requestId
? `/article/sr/${uploadInfo.requestId}`
: undefined
file.message = uploadInfo.message
setUploadFiles([...allFiles])
} catch (error) {
file.status = 'error'
setUploadFiles([...allFiles])
}
})()
}
})()
},
[uploadFiles]
)
@ -681,7 +681,14 @@ const UploadPad = (props: UploadPadProps): JSX.Element => {
</VStack>
</DragnDropIndicator>
</DragnDropStyle>
<VStack css={{ width: '100%', mt: '10px', gap: '5px', overflowY: 'auto' }}>
<VStack
css={{
width: '100%',
mt: '10px',
gap: '5px',
overflowY: 'auto',
}}
>
{uploadFiles.map((file) => {
return (
<HStack
@ -694,7 +701,7 @@ const UploadPad = (props: UploadPadProps): JSX.Element => {
padding: '15px',
gap: '10px',
color: '$thTextContrast',
overflow: "hidden"
overflow: 'hidden',
}}
alignment="center"
distribution="start"

5
packages/web/components/templates/UploadModal.tsx
View File

@ -293,7 +293,8 @@ export function UploadModal(props: UploadModalProps): JSX.Element {
title="Upload file"
onOpenChange={props.onOpenChange}
/>
<Dropzone
The uploader is currently disabled.
{/* <Dropzone
ref={dropzoneRef}
onDragEnter={() => {
setInDragOperation(true)
@ -447,7 +448,7 @@ export function UploadModal(props: UploadModalProps): JSX.Element {
<input {...getInputProps()} />
</div>
)}
</Dropzone>
</Dropzone> */}
</VStack>
</ModalContent>
</ModalRoot>

104
packages/web/pages/settings/integrations.tsx
View File

@ -237,42 +237,42 @@ export default function Integrations(): JSX.Element {
},
},
},
{
icon: '/static/icons/pocket.svg',
title: 'Pocket',
subText:
'Pocket is a place to save articles, videos, and more. Our Pocket integration allows importing your Pocket library to Omnivore. Once connected we will asyncronously import all your Pocket articles into Omnivore, as this process is resource intensive it can take some time. You will receive an email when the process is completed. Limit 20k articles per import. The import is a one-time process and can only be performed once per-account.',
button: {
text: pocket ? 'Disconnect' : 'Import',
icon: isImporting(pocket) ? (
<Spinner size={16} />
) : (
<Link size={16} weight={'bold'} />
),
style: pocket ? 'ctaWhite' : 'ctaDarkYellow',
action: () => {
pocket
? deleteIntegration(pocket.id)
: redirectToIntegration('POCKET', ImportItemState.Unarchived)
},
disabled: isImporting(pocket),
isDropdown: !pocket,
dropdownOptions: [
{
text: 'Import All',
action: () => {
redirectToIntegration('POCKET', ImportItemState.All)
},
},
{
text: 'Import Unarchived',
action: () => {
redirectToIntegration('POCKET', ImportItemState.Unarchived)
},
},
],
},
},
// {
// icon: '/static/icons/pocket.svg',
// title: 'Pocket',
// subText:
// 'Pocket is a place to save articles, videos, and more. Our Pocket integration allows importing your Pocket library to Omnivore. Once connected we will asyncronously import all your Pocket articles into Omnivore, as this process is resource intensive it can take some time. You will receive an email when the process is completed. Limit 20k articles per import. The import is a one-time process and can only be performed once per-account.',
// button: {
// text: pocket ? 'Disconnect' : 'Import',
// icon: isImporting(pocket) ? (
// <Spinner size={16} />
// ) : (
// <Link size={16} weight={'bold'} />
// ),
// style: pocket ? 'ctaWhite' : 'ctaDarkYellow',
// action: () => {
// pocket
// ? deleteIntegration(pocket.id)
// : redirectToIntegration('POCKET', ImportItemState.Unarchived)
// },
// disabled: isImporting(pocket),
// isDropdown: !pocket,
// dropdownOptions: [
// {
// text: 'Import All',
// action: () => {
// redirectToIntegration('POCKET', ImportItemState.All)
// },
// },
// {
// text: 'Import Unarchived',
// action: () => {
// redirectToIntegration('POCKET', ImportItemState.Unarchived)
// },
// },
// ],
// },
// },
// {
// icon: '/static/icons/webhooks.svg',
@ -301,22 +301,22 @@ export default function Integrations(): JSX.Element {
},
},
},
{
icon: '/static/icons/notion.png',
title: 'Notion',
subText:
'Notion is an all-in-one workspace. Use our Notion integration to sync your Omnivore items to Notion.',
button: {
text: notion ? 'Settings' : 'Connect',
icon: <Link size={16} weight={'bold'} />,
style: notion ? 'ctaWhite' : 'ctaDarkYellow',
action: () => {
notion
? router.push('/settings/integrations/notion')
: redirectToIntegration('NOTION')
},
},
},
// {
// icon: '/static/icons/notion.png',
// title: 'Notion',
// subText:
// 'Notion is an all-in-one workspace. Use our Notion integration to sync your Omnivore items to Notion.',
// button: {
// text: notion ? 'Settings' : 'Connect',
// icon: <Link size={16} weight={'bold'} />,
// style: notion ? 'ctaWhite' : 'ctaDarkYellow',
// action: () => {
// notion
// ? router.push('/settings/integrations/notion')
// : redirectToIntegration('NOTION')
// },
// },
// },
]
setIntegrationsArray(integrationsArray)