From 9314c3d8f164f013e839111da592e32cebc4827e Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 19 Dec 2022 12:42:32 +0800 Subject: [PATCH] Add uploadImportFile API method Add uploadImportFile API method Fix prefix, counting max files uploaded Add resolver types Basic web ui for the uploader interface Allow selecting type when uploading import files --- packages/api/src/generated/graphql.ts | 56 +++++++++ packages/api/src/generated/schema.graphql | 22 ++++ .../api/src/resolvers/function_resolvers.ts | 3 + .../importers/uploadImportFileResolver.ts | 93 ++++++++++++++ packages/api/src/schema.ts | 25 ++++ packages/api/src/utils/uploads.ts | 5 + .../mutations/uploadImportFileMutation.ts | 40 ++++++ packages/web/pages/tools/import/upload.tsx | 119 ++++++++++++++++++ 8 files changed, 363 insertions(+) create mode 100644 packages/api/src/resolvers/importers/uploadImportFileResolver.ts create mode 100644 packages/web/lib/networking/mutations/uploadImportFileMutation.ts create mode 100644 packages/web/pages/tools/import/upload.tsx diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 587a7488c..aa60707c9 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1209,6 +1209,7 @@ export type Mutation = { updateUser: UpdateUserResult; updateUserProfile: UpdateUserProfileResult; uploadFileRequest: UploadFileRequestResult; + uploadImportFile: UploadImportFileResult; }; @@ -1521,6 +1522,12 @@ export type MutationUploadFileRequestArgs = { input: UploadFileRequestInput; }; + +export type MutationUploadImportFileArgs = { + contentType: Scalars['String']; + type: UploadImportFileType; +}; + export type NewsletterEmail = { __typename?: 'NewsletterEmail'; address: Scalars['String']; @@ -2884,6 +2891,29 @@ export enum UploadFileStatus { Initialized = 'INITIALIZED' } +export type UploadImportFileError = { + __typename?: 'UploadImportFileError'; + errorCodes: Array; +}; + +export enum UploadImportFileErrorCode { + BadRequest = 'BAD_REQUEST', + Unauthorized = 'UNAUTHORIZED', + UploadDailyLimitExceeded = 'UPLOAD_DAILY_LIMIT_EXCEEDED' +} + +export type UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess; + +export type UploadImportFileSuccess = { + __typename?: 'UploadImportFileSuccess'; + uploadSignedUrl?: Maybe; +}; + +export enum UploadImportFileType { + Pocket = 'POCKET', + UrlList = 'URL_LIST' +} + export type User = { __typename?: 'User'; followersCount?: Maybe; @@ -3518,6 +3548,11 @@ export type ResolversTypes = { UploadFileRequestResult: ResolversTypes['UploadFileRequestError'] | ResolversTypes['UploadFileRequestSuccess']; UploadFileRequestSuccess: ResolverTypeWrapper; UploadFileStatus: UploadFileStatus; + UploadImportFileError: ResolverTypeWrapper; + UploadImportFileErrorCode: UploadImportFileErrorCode; + UploadImportFileResult: ResolversTypes['UploadImportFileError'] | ResolversTypes['UploadImportFileSuccess']; + UploadImportFileSuccess: ResolverTypeWrapper; + UploadImportFileType: UploadImportFileType; User: ResolverTypeWrapper; UserError: ResolverTypeWrapper; UserErrorCode: UserErrorCode; @@ -3886,6 +3921,9 @@ export type ResolversParentTypes = { UploadFileRequestInput: UploadFileRequestInput; UploadFileRequestResult: ResolversParentTypes['UploadFileRequestError'] | ResolversParentTypes['UploadFileRequestSuccess']; UploadFileRequestSuccess: UploadFileRequestSuccess; + UploadImportFileError: UploadImportFileError; + UploadImportFileResult: ResolversParentTypes['UploadImportFileError'] | ResolversParentTypes['UploadImportFileSuccess']; + UploadImportFileSuccess: UploadImportFileSuccess; User: User; UserError: UserError; UserPersonalization: UserPersonalization; @@ -4805,6 +4843,7 @@ export type MutationResolvers>; updateUserProfile?: Resolver>; uploadFileRequest?: Resolver>; + uploadImportFile?: Resolver>; }; export type NewsletterEmailResolvers = { @@ -5607,6 +5646,20 @@ export type UploadFileRequestSuccessResolvers; }; +export type UploadImportFileErrorResolvers = { + errorCodes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + +export type UploadImportFileResultResolvers = { + __resolveType: TypeResolveFn<'UploadImportFileError' | 'UploadImportFileSuccess', ParentType, ContextType>; +}; + +export type UploadImportFileSuccessResolvers = { + uploadSignedUrl?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}; + export type UserResolvers = { followersCount?: Resolver, ParentType, ContextType>; friendsCount?: Resolver, ParentType, ContextType>; @@ -5996,6 +6049,9 @@ export type Resolvers = { UploadFileRequestError?: UploadFileRequestErrorResolvers; UploadFileRequestResult?: UploadFileRequestResultResolvers; UploadFileRequestSuccess?: UploadFileRequestSuccessResolvers; + UploadImportFileError?: UploadImportFileErrorResolvers; + UploadImportFileResult?: UploadImportFileResultResolvers; + UploadImportFileSuccess?: UploadImportFileSuccessResolvers; User?: UserResolvers; UserError?: UserErrorResolvers; UserPersonalization?: UserPersonalizationResolvers; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 81193a90f..6f81fc062 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1083,6 +1083,7 @@ type Mutation { updateUser(input: UpdateUserInput!): UpdateUserResult! updateUserProfile(input: UpdateUserProfileInput!): UpdateUserProfileResult! uploadFileRequest(input: UploadFileRequestInput!): UploadFileRequestResult! + uploadImportFile(contentType: String!, type: UploadImportFileType!): UploadImportFileResult! } type NewsletterEmail { @@ -2251,6 +2252,27 @@ enum UploadFileStatus { INITIALIZED } +type UploadImportFileError { + errorCodes: [UploadImportFileErrorCode!]! +} + +enum UploadImportFileErrorCode { + BAD_REQUEST + UNAUTHORIZED + UPLOAD_DAILY_LIMIT_EXCEEDED +} + +union UploadImportFileResult = UploadImportFileError | UploadImportFileSuccess + +type UploadImportFileSuccess { + uploadSignedUrl: String +} + +enum UploadImportFileType { + POCKET + URL_LIST +} + type User { followersCount: Int friendsCount: Int diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index 85255a93d..39dac70c3 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -116,6 +116,7 @@ import { import { getPageByParam } from '../elastic/pages' import { recentSearchesResolver } from './recent_searches' import { optInFeatureResolver } from './features' +import { uploadImportFileResolver } from './importers/uploadImportFileResolver' /* eslint-disable @typescript-eslint/naming-convention */ type ResultResolveType = { @@ -197,6 +198,7 @@ export const functionResolvers = { joinGroup: joinGroupResolver, recommendHighlights: recommendHighlightsResolver, leaveGroup: leaveGroupResolver, + uploadImportFile: uploadImportFileResolver, }, Query: { me: getMeUserResolver, @@ -656,4 +658,5 @@ export const functionResolvers = { ...resultResolveTypeResolver('JoinGroup'), ...resultResolveTypeResolver('RecommendHighlights'), ...resultResolveTypeResolver('LeaveGroup'), + ...resultResolveTypeResolver('UploadImportFile'), } diff --git a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts new file mode 100644 index 000000000..544f97dc6 --- /dev/null +++ b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts @@ -0,0 +1,93 @@ +import { authorized } from '../../utils/helpers' +import { + UploadImportFileErrorCode, + MutationUploadImportFileArgs, + UploadImportFileError, + UploadImportFileSuccess, +} from '../../generated/graphql' +import { getRepository } from '../../entity/utils' +import { User } from '../../entity/user' +import { analytics } from '../../utils/analytics' +import { env } from '../../env' +import { DateTime } from 'luxon' +import { + countOfFilesWithPrefix, + generateUploadSignedUrl, +} from '../../utils/uploads' +import { v4 as uuidv4 } from 'uuid' +import { buildLogger } from '../../utils/logger' + +const MAX_DAILY_UPLOADS = 4 +const VALID_CONTENT_TYPES = ['text/csv'] + +const logger = buildLogger('app.dispatch') + +const extensionForContentType = (contentType: string) => { + switch (contentType) { + case 'text/csv': + return 'csv' + } + return '.unknown' +} + +export const uploadImportFileResolver = authorized< + UploadImportFileSuccess, + UploadImportFileError, + MutationUploadImportFileArgs +>(async (_, { type, contentType }, { claims: { uid }, log }) => { + log.info('uploadImportFileResolver') + + if (!VALID_CONTENT_TYPES.includes(contentType)) { + return { + errorCodes: [UploadImportFileErrorCode.BadRequest], + } + } + + const user = await getRepository(User).findOneBy({ id: uid }) + if (!user) { + return { + errorCodes: [UploadImportFileErrorCode.Unauthorized], + } + } + + analytics.track({ + userId: uid, + event: 'upload_import_file', + properties: { + type, + env: env.server.apiEnv, + }, + }) + + // path style: imports///- + const dateStr = DateTime.now().toISODate() + const dirPath = `imports/${uid}/${dateStr}/` + const fileCount = await countOfFilesWithPrefix(dirPath) + + if (fileCount > MAX_DAILY_UPLOADS) { + return { + errorCodes: [UploadImportFileErrorCode.UploadDailyLimitExceeded], + } + } + + try { + const fileUuid = uuidv4() + const ext = extensionForContentType(contentType) + const fullPath = `${dirPath}${type}-${fileUuid}.${ext}` + const uploadSignedUrl = await generateUploadSignedUrl(fullPath, contentType) + + return { + uploadSignedUrl, + } + } catch (error) { + logger.error('Error creating uploadSignedUrl', { + error, + type, + contentType, + }) + + return { + errorCodes: [UploadImportFileErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 281749da2..ba9423d61 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2306,6 +2306,27 @@ const schema = gql` NOT_FOUND } + enum UploadImportFileType { + URL_LIST + POCKET + } + + enum UploadImportFileErrorCode { + UNAUTHORIZED + BAD_REQUEST + UPLOAD_DAILY_LIMIT_EXCEEDED + } + + union UploadImportFileResult = UploadImportFileSuccess | UploadImportFileError + + type UploadImportFileError { + errorCodes: [UploadImportFileErrorCode!]! + } + + type UploadImportFileSuccess { + uploadSignedUrl: String + } + # Mutations type Mutation { googleLogin(input: GoogleLoginInput!): LoginResult! @@ -2390,6 +2411,10 @@ const schema = gql` input: RecommendHighlightsInput! ): RecommendHighlightsResult! leaveGroup(groupId: ID!): LeaveGroupResult! + uploadImportFile( + type: UploadImportFileType! + contentType: String! + ): UploadImportFileResult! } # FIXME: remove sort from feedArticles after all cached tabs are closed diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 8e631f3e1..814acc143 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -18,6 +18,11 @@ export const getFilePublicUrl = (filePathName: string): string => { return storage.bucket(bucketName).file(filePathName).publicUrl() } +export const countOfFilesWithPrefix = async (prefix: string) => { + const [files] = await storage.bucket(bucketName).getFiles({ prefix }) + return files.length +} + export const generateUploadSignedUrl = async ( filePathName: string, contentType: string, diff --git a/packages/web/lib/networking/mutations/uploadImportFileMutation.ts b/packages/web/lib/networking/mutations/uploadImportFileMutation.ts new file mode 100644 index 000000000..e272906b0 --- /dev/null +++ b/packages/web/lib/networking/mutations/uploadImportFileMutation.ts @@ -0,0 +1,40 @@ +import { gqlFetcher } from '../networkHelpers' +import { v4 as uuidv4 } from 'uuid' + +export type UploadImportFileType = 'URL_LIST' | 'POCKET' + +type UploadImportFileResponseData = { + uploadImportFile?: UploadImportFileData +} + +type UploadImportFileData = { + uploadSignedUrl: string + errorCodes?: unknown[] +} + +export async function uploadImportFileRequestMutation( + type: UploadImportFileType, + contentType: string +): Promise { + const mutation = ` + mutation UploadImportFile($type: UploadImportFileType!, $contentType: String!) { + uploadImportFile(type:$type, contentType:$contentType) { + ... on UploadImportFileError { + errorCodes + } + ... on UploadImportFileSuccess { + uploadSignedUrl + } + } + }` + + const data = await gqlFetcher(mutation, { type, contentType }) + console.log('UploadImportFile: ', data) + const output = data as UploadImportFileResponseData | undefined + const error = output?.uploadImportFile?.errorCodes?.find(() => true) + console.log('error: ', error) + if (error) { + throw error + } + return output?.uploadImportFile +} diff --git a/packages/web/pages/tools/import/upload.tsx b/packages/web/pages/tools/import/upload.tsx new file mode 100644 index 000000000..23a9bc991 --- /dev/null +++ b/packages/web/pages/tools/import/upload.tsx @@ -0,0 +1,119 @@ +import { useCallback, useMemo, useState } from 'react' +import { Toaster } from 'react-hot-toast' + +import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers' +import { applyStoredTheme } from '../../../lib/themeUpdater' + +import { + Box, + HStack, + SpanBox, + VStack, +} from '../../../components/elements/LayoutPrimitives' +import { PrimaryLayout } from '../../../components/templates/PrimaryLayout' + +import 'antd/dist/antd.compact.css' +import { StyledText } from '../../../components/elements/StyledText' +import { Button } from '../../../components/elements/Button' +import { ProfileLayout } from '../../../components/templates/ProfileLayout' +import { FormLabel } from '../../../components/elements/FormElements' +import { uploadImportFileRequestMutation } from '../../../lib/networking/mutations/uploadImportFileMutation' + +export default function ImportUploader(): JSX.Element { + applyStoredTheme(false) + + const [errorMessage, setErrorMessage] = useState() + const [file, setFile] = useState() + const [type, setType] = useState() + + const handleFileChange = (e: ChangeEvent) => { + if (e.target.files) { + setFile(e.target.files[0]) + } + } + + const handleUploadClick = async () => { + if (!file) { + return + } + + console.log('file type: ', file.type) + + try { + const result = await uploadImportFileRequestMutation( + 'URL_LIST', + 'text/csv' + ) + + if (result && result.uploadSignedUrl) { + const uploadRes = await fetch(result.uploadSignedUrl, { + method: 'PUT', + body: file, + headers: { + 'content-type': 'text/csv', + 'content-length': `${file.size}`, + }, + }) + console.log('upload result: ', uploadRes) + } + } catch (error) { + console.log('caught error', error) + if (error == 'UPLOAD_DAILY_LIMIT_EXCEEDED') { + setErrorMessage('You have exceeded your maximum daily upload limit.') + } + } + } + + return ( + + + + + Type + + + + + File + + +
{file && `${file.name} - ${file.type}`}
+ + + + {errorMessage && {errorMessage}} + + + + ) +}