Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add uploadImportFile API method

Browse Source 
Add uploadImportFile API method

Fix prefix, counting max files uploaded

Add resolver types

Basic web ui for the uploader interface

Allow selecting type when uploading import files
This commit is contained in:
Jackson Harper
2022-12-19 12:42:32 +08:00
parent 08f3cb9e72
commit 9314c3d8f1
8 changed files with 363 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
packages/api/src/generated/graphql.ts
View File

@ -1209,6 +1209,7 @@ export type Mutation = {
updateUser: UpdateUserResult;
updateUserProfile: UpdateUserProfileResult;
uploadFileRequest: UploadFileRequestResult;
uploadImportFile: UploadImportFileResult;
};
@ -1521,6 +1522,12 @@ export type MutationUploadFileRequestArgs = {
input: UploadFileRequestInput;
};
export type MutationUploadImportFileArgs = {
contentType: Scalars['String'];
type: UploadImportFileType;
};
export type NewsletterEmail = {
__typename?: 'NewsletterEmail';
address: Scalars['String'];
@ -2884,6 +2891,29 @@ export enum UploadFileStatus {
Initialized = 'INITIALIZED'
}
export type UploadImportFileError = {
__typename?: 'UploadImportFileError';
errorCodes: Array<UploadImportFileErrorCode>;
};
export enum UploadImportFileErrorCode {
BadRequest = 'BAD_REQUEST',
Unauthorized = 'UNAUTHORIZED',
UploadDailyLimitExceeded = 'UPLOAD_DAILY_LIMIT_EXCEEDED'
}
export type UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess;
export type UploadImportFileSuccess = {
__typename?: 'UploadImportFileSuccess';
uploadSignedUrl?: Maybe<Scalars['String']>;
};
export enum UploadImportFileType {
Pocket = 'POCKET',
UrlList = 'URL_LIST'
}
export type User = {
__typename?: 'User';
followersCount?: Maybe<Scalars['Int']>;
@ -3518,6 +3548,11 @@ export type ResolversTypes = {
UploadFileRequestResult: ResolversTypes['UploadFileRequestError'] | ResolversTypes['UploadFileRequestSuccess'];
UploadFileRequestSuccess: ResolverTypeWrapper<UploadFileRequestSuccess>;
UploadFileStatus: UploadFileStatus;
UploadImportFileError: ResolverTypeWrapper<UploadImportFileError>;
UploadImportFileErrorCode: UploadImportFileErrorCode;
UploadImportFileResult: ResolversTypes['UploadImportFileError'] | ResolversTypes['UploadImportFileSuccess'];
UploadImportFileSuccess: ResolverTypeWrapper<UploadImportFileSuccess>;
UploadImportFileType: UploadImportFileType;
User: ResolverTypeWrapper<User>;
UserError: ResolverTypeWrapper<UserError>;
UserErrorCode: UserErrorCode;
@ -3886,6 +3921,9 @@ export type ResolversParentTypes = {
UploadFileRequestInput: UploadFileRequestInput;
UploadFileRequestResult: ResolversParentTypes['UploadFileRequestError'] | ResolversParentTypes['UploadFileRequestSuccess'];
UploadFileRequestSuccess: UploadFileRequestSuccess;
UploadImportFileError: UploadImportFileError;
UploadImportFileResult: ResolversParentTypes['UploadImportFileError'] | ResolversParentTypes['UploadImportFileSuccess'];
UploadImportFileSuccess: UploadImportFileSuccess;
User: User;
UserError: UserError;
UserPersonalization: UserPersonalization;
@ -4805,6 +4843,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
updateUser?: Resolver<ResolversTypes['UpdateUserResult'], ParentType, ContextType, RequireFields<MutationUpdateUserArgs, 'input'>>;
updateUserProfile?: Resolver<ResolversTypes['UpdateUserProfileResult'], ParentType, ContextType, RequireFields<MutationUpdateUserProfileArgs, 'input'>>;
uploadFileRequest?: Resolver<ResolversTypes['UploadFileRequestResult'], ParentType, ContextType, RequireFields<MutationUploadFileRequestArgs, 'input'>>;
uploadImportFile?: Resolver<ResolversTypes['UploadImportFileResult'], ParentType, ContextType, RequireFields<MutationUploadImportFileArgs, 'contentType' | 'type'>>;
};
export type NewsletterEmailResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['NewsletterEmail'] = ResolversParentTypes['NewsletterEmail']> = {
@ -5607,6 +5646,20 @@ export type UploadFileRequestSuccessResolvers<ContextType = ResolverContext, Par
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type UploadImportFileErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['UploadImportFileError'] = ResolversParentTypes['UploadImportFileError']> = {
errorCodes?: Resolver<Array<ResolversTypes['UploadImportFileErrorCode']>, ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type UploadImportFileResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['UploadImportFileResult'] = ResolversParentTypes['UploadImportFileResult']> = {
__resolveType: TypeResolveFn<'UploadImportFileError' | 'UploadImportFileSuccess', ParentType, ContextType>;
};
export type UploadImportFileSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['UploadImportFileSuccess'] = ResolversParentTypes['UploadImportFileSuccess']> = {
uploadSignedUrl?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type UserResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['User'] = ResolversParentTypes['User']> = {
followersCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
friendsCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
@ -5996,6 +6049,9 @@ export type Resolvers<ContextType = ResolverContext> = {
UploadFileRequestError?: UploadFileRequestErrorResolvers<ContextType>;
UploadFileRequestResult?: UploadFileRequestResultResolvers<ContextType>;
UploadFileRequestSuccess?: UploadFileRequestSuccessResolvers<ContextType>;
UploadImportFileError?: UploadImportFileErrorResolvers<ContextType>;
UploadImportFileResult?: UploadImportFileResultResolvers<ContextType>;
UploadImportFileSuccess?: UploadImportFileSuccessResolvers<ContextType>;
User?: UserResolvers<ContextType>;
UserError?: UserErrorResolvers<ContextType>;
UserPersonalization?: UserPersonalizationResolvers<ContextType>;

22
packages/api/src/generated/schema.graphql
View File

@ -1083,6 +1083,7 @@ type Mutation {
updateUser(input: UpdateUserInput!): UpdateUserResult!
updateUserProfile(input: UpdateUserProfileInput!): UpdateUserProfileResult!
uploadFileRequest(input: UploadFileRequestInput!): UploadFileRequestResult!
uploadImportFile(contentType: String!, type: UploadImportFileType!): UploadImportFileResult!
}
type NewsletterEmail {
@ -2251,6 +2252,27 @@ enum UploadFileStatus {
INITIALIZED
}
type UploadImportFileError {
errorCodes: [UploadImportFileErrorCode!]!
}
enum UploadImportFileErrorCode {
BAD_REQUEST
UNAUTHORIZED
UPLOAD_DAILY_LIMIT_EXCEEDED
}
union UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess
type UploadImportFileSuccess {
uploadSignedUrl: String
}
enum UploadImportFileType {
POCKET
URL_LIST
}
type User {
followersCount: Int
friendsCount: Int

3
packages/api/src/resolvers/function_resolvers.ts
View File

@ -116,6 +116,7 @@ import {
import { getPageByParam } from '../elastic/pages'
import { recentSearchesResolver } from './recent_searches'
import { optInFeatureResolver } from './features'
import { uploadImportFileResolver } from './importers/uploadImportFileResolver'
/* eslint-disable @typescript-eslint/naming-convention */
type ResultResolveType = {
@ -197,6 +198,7 @@ export const functionResolvers = {
joinGroup: joinGroupResolver,
recommendHighlights: recommendHighlightsResolver,
leaveGroup: leaveGroupResolver,
uploadImportFile: uploadImportFileResolver,
},
Query: {
me: getMeUserResolver,
@ -656,4 +658,5 @@ export const functionResolvers = {
...resultResolveTypeResolver('JoinGroup'),
...resultResolveTypeResolver('RecommendHighlights'),
...resultResolveTypeResolver('LeaveGroup'),
...resultResolveTypeResolver('UploadImportFile'),
}

93
packages/api/src/resolvers/importers/uploadImportFileResolver.ts Normal file
View File

@ -0,0 +1,93 @@
import { authorized } from '../../utils/helpers'
import {
UploadImportFileErrorCode,
MutationUploadImportFileArgs,
UploadImportFileError,
UploadImportFileSuccess,
} from '../../generated/graphql'
import { getRepository } from '../../entity/utils'
import { User } from '../../entity/user'
import { analytics } from '../../utils/analytics'
import { env } from '../../env'
import { DateTime } from 'luxon'
import {
countOfFilesWithPrefix,
generateUploadSignedUrl,
} from '../../utils/uploads'
import { v4 as uuidv4 } from 'uuid'
import { buildLogger } from '../../utils/logger'
const MAX_DAILY_UPLOADS = 4
const VALID_CONTENT_TYPES = ['text/csv']
const logger = buildLogger('app.dispatch')
const extensionForContentType = (contentType: string) => {
switch (contentType) {
case 'text/csv':
return 'csv'
}
return '.unknown'
}
export const uploadImportFileResolver = authorized<
UploadImportFileSuccess,
UploadImportFileError,
MutationUploadImportFileArgs
>(async (_, { type, contentType }, { claims: { uid }, log }) => {
log.info('uploadImportFileResolver')
if (!VALID_CONTENT_TYPES.includes(contentType)) {
return {
errorCodes: [UploadImportFileErrorCode.BadRequest],
}
}
const user = await getRepository(User).findOneBy({ id: uid })
if (!user) {
return {
errorCodes: [UploadImportFileErrorCode.Unauthorized],
}
}
analytics.track({
userId: uid,
event: 'upload_import_file',
properties: {
type,
env: env.server.apiEnv,
},
})
// path style: imports/<uid>/<date>/<type>-<uuid>
const dateStr = DateTime.now().toISODate()
const dirPath = `imports/${uid}/${dateStr}/`
const fileCount = await countOfFilesWithPrefix(dirPath)
if (fileCount > MAX_DAILY_UPLOADS) {
return {
errorCodes: [UploadImportFileErrorCode.UploadDailyLimitExceeded],
}
}
try {
const fileUuid = uuidv4()
const ext = extensionForContentType(contentType)
const fullPath = `${dirPath}${type}-${fileUuid}.${ext}`
const uploadSignedUrl = await generateUploadSignedUrl(fullPath, contentType)
return {
uploadSignedUrl,
}
} catch (error) {
logger.error('Error creating uploadSignedUrl', {
error,
type,
contentType,
})
return {
errorCodes: [UploadImportFileErrorCode.BadRequest],
}
}
})

25
packages/api/src/schema.ts
View File

@ -2306,6 +2306,27 @@ const schema = gql`
NOT_FOUND
}
enum UploadImportFileType {
URL_LIST
POCKET
}
enum UploadImportFileErrorCode {
UNAUTHORIZED
BAD_REQUEST
UPLOAD_DAILY_LIMIT_EXCEEDED
}
union UploadImportFileResult = UploadImportFileSuccess | UploadImportFileError
type UploadImportFileError {
errorCodes: [UploadImportFileErrorCode!]!
}
type UploadImportFileSuccess {
uploadSignedUrl: String
}
# Mutations
type Mutation {
googleLogin(input: GoogleLoginInput!): LoginResult!
@ -2390,6 +2411,10 @@ const schema = gql`
input: RecommendHighlightsInput!
): RecommendHighlightsResult!
leaveGroup(groupId: ID!): LeaveGroupResult!
uploadImportFile(
type: UploadImportFileType!
contentType: String!
): UploadImportFileResult!
}
# FIXME: remove sort from feedArticles after all cached tabs are closed

5
packages/api/src/utils/uploads.ts
View File

@ -18,6 +18,11 @@ export const getFilePublicUrl = (filePathName: string): string => {
return storage.bucket(bucketName).file(filePathName).publicUrl()
}
export const countOfFilesWithPrefix = async (prefix: string) => {
const [files] = await storage.bucket(bucketName).getFiles({ prefix })
return files.length
}
export const generateUploadSignedUrl = async (
filePathName: string,
contentType: string,

40
packages/web/lib/networking/mutations/uploadImportFileMutation.ts Normal file
View File

@ -0,0 +1,40 @@
import { gqlFetcher } from '../networkHelpers'
import { v4 as uuidv4 } from 'uuid'
export type UploadImportFileType = 'URL_LIST' | 'POCKET'
type UploadImportFileResponseData = {
uploadImportFile?: UploadImportFileData
}
type UploadImportFileData = {
uploadSignedUrl: string
errorCodes?: unknown[]
}
export async function uploadImportFileRequestMutation(
type: UploadImportFileType,
contentType: string
): Promise<UploadImportFileData | undefined> {
const mutation = `
mutation UploadImportFile($type: UploadImportFileType!, $contentType: String!) {
uploadImportFile(type:$type, contentType:$contentType) {
... on UploadImportFileError {
errorCodes
}
... on UploadImportFileSuccess {
uploadSignedUrl
}
}
}`
const data = await gqlFetcher(mutation, { type, contentType })
console.log('UploadImportFile: ', data)
const output = data as UploadImportFileResponseData | undefined
const error = output?.uploadImportFile?.errorCodes?.find(() => true)
console.log('error: ', error)
if (error) {
throw error
}
return output?.uploadImportFile
}

119
packages/web/pages/tools/import/upload.tsx Normal file
View File

@ -0,0 +1,119 @@
import { useCallback, useMemo, useState } from 'react'
import { Toaster } from 'react-hot-toast'
import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers'
import { applyStoredTheme } from '../../../lib/themeUpdater'
import {
Box,
HStack,
SpanBox,
VStack,
} from '../../../components/elements/LayoutPrimitives'
import { PrimaryLayout } from '../../../components/templates/PrimaryLayout'
import 'antd/dist/antd.compact.css'
import { StyledText } from '../../../components/elements/StyledText'
import { Button } from '../../../components/elements/Button'
import { ProfileLayout } from '../../../components/templates/ProfileLayout'
import { FormLabel } from '../../../components/elements/FormElements'
import { uploadImportFileRequestMutation } from '../../../lib/networking/mutations/uploadImportFileMutation'
export default function ImportUploader(): JSX.Element {
applyStoredTheme(false)
const [errorMessage, setErrorMessage] = useState<string | undefined>()
const [file, setFile] = useState<File>()
const [type, setType] = useState<string>()
const handleFileChange = (e: ChangeEvent<HTMLInputElement>) => {
if (e.target.files) {
setFile(e.target.files[0])
}
}
const handleUploadClick = async () => {
if (!file) {
return
}
console.log('file type: ', file.type)
try {
const result = await uploadImportFileRequestMutation(
'URL_LIST',
'text/csv'
)
if (result && result.uploadSignedUrl) {
const uploadRes = await fetch(result.uploadSignedUrl, {
method: 'PUT',
body: file,
headers: {
'content-type': 'text/csv',
'content-length': `${file.size}`,
},
})
console.log('upload result: ', uploadRes)
}
} catch (error) {
console.log('caught error', error)
if (error == 'UPLOAD_DAILY_LIMIT_EXCEEDED') {
setErrorMessage('You have exceeded your maximum daily upload limit.')
}
}
}
return (
<ProfileLayout>
<VStack
alignment="center"
css={{
padding: '16px',
background: 'white',
minWidth: '340px',
width: '70vw',
maxWidth: '576px',
borderRadius: '8px',
boxShadow: 'rgb(224 224 224) 9px 9px 9px -9px',
}}
>
<VStack
css={{ width: '100%', minWidth: '320px', gap: '16px', pb: '16px' }}
></VStack>
<SpanBox css={{ width: '100%' }}>
<FormLabel>Type</FormLabel>
<select
onChange={(event) => setType('')}
style={{
padding: '8px',
height: '38px',
borderRadius: '6px',
minWidth: '196px',
}}
>
<option value="URL_LIST">CSV of URLs</option>
<option value="POCKET">Pocket export file</option>
</select>
</SpanBox>
<SpanBox css={{ width: '100%' }}>
<FormLabel>File</FormLabel>
<HStack css={{ py: '16px' }} distribution="center">
<input type="file" onChange={handleFileChange} />
<div>{file && `${file.name} - ${file.type}`}</div>
</HStack>
</SpanBox>
{errorMessage && <StyledText style="error">{errorMessage}</StyledText>}
<Button
onClick={handleUploadClick}
style="ctaDarkYellow"
css={{ my: '$2' }}
>
Upload File
</Button>
</VStack>
</ProfileLayout>
)
}