Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove article saving request (#493)

Browse Source 
* Add state and taskName in elastic page mappings

* Add state and taskName in elastic page interface

* Create page with PROCESSING state before scrapping

* Update createArticleRequest API

* Fix tests

* Add default state for pages

* Update createArticle API

* Update save page

* Update save file

* Update saving item description

* Show unable to parse content for failed page

* Fix date parsing

* Search for not failed pages

* Fix tests

* Add test for saveUrl

* Update get article saving request api

* Update get article test

* Add test for articleSavingRequest API

* Add test for failure

* Return new page id if clientRequestId empty

* Update clientRequestId in savePage

* Update clientRequestId in saveFile

* Replace article with slug in articleSavingRequest

* Add slug in articleSavingRequest response

* Depreciate article

* Use slug in web

* Remove article and highlight fragments

* Query article.slug on Prod

* Show unable to parse description for failed page

* Fix a bug having duplicate pages when saving the same url multiple times

* Add state in response

* Rename variables in removeArticle API

* Rename state

* Add state in response in web

* Make state an enum

* Open temporary page by link id

* Use an empty reader view as the background for loading pages

* Progressively load the article page as content is loaded

* Add includePending flag in getArticles API

* Set includePending = true in web

* Add elastic update mappings in migration script

* Add elastic mappings in docker image

* Move index_settings.json to migrate package

* Remove elastic index creation in api

* Move elastic migrations to a separate directory

* Remove index_settings from api docker image

Co-authored-by: Jackson Harper <jacksonh@gmail.com>
This commit is contained in:
Hongbo Wu
2022-04-29 13:41:06 +08:00
committed by GitHub
parent 170b7d9144
commit 2b70d480d2
42 changed files with 1101 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/api/Dockerfile
View File

@ -40,7 +40,6 @@ COPY --from=builder /app/packages/api/package.json /app/packages/api/package.jso
COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules
COPY --from=builder /app/node_modules /app/node_modules
COPY --from=builder /app/package.json /app/package.json
COPY --from=builder /app/packages/api/index_settings.json /app/packages/api/index_settings.json
EXPOSE 8080
CMD ["yarn", "workspace", "@omnivore/api", "start"]

20
packages/api/src/elastic/index.ts
View File

@ -1,7 +1,5 @@
import { env } from '../env'
import { Client } from '@elastic/elasticsearch'
import { readFileSync } from 'fs'
import { join } from 'path'
export const INDEX_NAME = 'pages'
export const INDEX_ALIAS = 'pages_alias'
@ -15,19 +13,6 @@ export const client = new Client({
},
})
const ingest = async (): Promise<void> => {
// read index settings from file
const indexSettings = readFileSync(
join(__dirname, '..', '..', 'index_settings.json'),
'utf8'
)
// create index
await client.indices.create({
index: INDEX_NAME,
body: indexSettings,
})
}
export const initElasticsearch = async (): Promise<void> => {
try {
const response = await client.info()
@ -38,10 +23,7 @@ export const initElasticsearch = async (): Promise<void> => {
index: INDEX_ALIAS,
})
if (!indexExists) {
console.log('ingesting index...')
await ingest()
await client.indices.refresh({ index: INDEX_ALIAS })
throw new Error('elastic index does not exist')
}
console.log('elastic client is ready')
} catch (e) {

18
packages/api/src/elastic/pages.ts
View File

@ -1,4 +1,5 @@
import {
ArticleSavingRequestStatus,
Page,
PageContext,
PageType,
@ -335,6 +336,7 @@ export const searchPages = async (
savedDateFilter?: DateRangeFilter
publishedDateFilter?: DateRangeFilter
subscriptionFilter?: SubscriptionFilter
includePending?: boolean | null
},
userId: string
): Promise<[Page[], number] | undefined> => {
@ -375,7 +377,13 @@ export const searchPages = async (
},
],
should: [],
must_not: [],
must_not: [
{
term: {
state: ArticleSavingRequestStatus.Failed,
},
},
],
},
},
sort: [
@ -424,6 +432,14 @@ export const searchPages = async (
appendSubscriptionFilter(body, subscriptionFilter)
}
if (!args.includePending) {
body.query.bool.must_not.push({
term: {
state: ArticleSavingRequestStatus.Processing,
},
})
}
console.log('searching pages in elastic', JSON.stringify(body))
const response = await client.search<SearchResponse<Page>, SearchBody>({

12
packages/api/src/elastic/types.ts
View File

@ -69,6 +69,7 @@ export interface SearchBody {
}[]
minimum_should_match?: number
must_not: (
| { term: { state: ArticleSavingRequestStatus } }
| {
exists: {
field: string
@ -147,6 +148,12 @@ export enum PageType {
Highlights = 'HIGHLIGHTS',
}
export enum ArticleSavingRequestStatus {
Failed = 'FAILED',
Processing = 'PROCESSING',
Succeeded = 'SUCCEEDED',
}
export interface Label {
id: string
name: string
@ -199,6 +206,8 @@ export interface Page {
subscription?: string
unsubMailTo?: string
unsubHttpUrl?: string
state: ArticleSavingRequestStatus
taskName?: string
}
export interface SearchItem {
@ -221,9 +230,10 @@ export interface SearchItem {
readingProgressPercent?: number
readingProgressAnchorIndex?: number
userId: string
state?: ArticleSavingRequestStatus
}
const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId'] as const
const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const
export type ParamSet = PickTuple<Page, typeof keys>

8
packages/api/src/generated/graphql.ts
View File

@ -69,6 +69,7 @@ export type Article = {
siteIcon?: Maybe<Scalars['String']>;
siteName?: Maybe<Scalars['String']>;
slug: Scalars['String'];
state?: Maybe<ArticleSavingRequestStatus>;
subscription?: Maybe<Scalars['String']>;
title: Scalars['String'];
unsubHttpUrl?: Maybe<Scalars['String']>;
@ -107,10 +108,12 @@ export type ArticleResult = ArticleError | ArticleSuccess;
export type ArticleSavingRequest = {
__typename?: 'ArticleSavingRequest';
/** @deprecated article has been replaced with slug */
article?: Maybe<Article>;
createdAt: Scalars['Date'];
errorCode?: Maybe<CreateArticleErrorCode>;
id: Scalars['ID'];
slug: Scalars['String'];
status: ArticleSavingRequestStatus;
updatedAt: Scalars['Date'];
user: User;
@ -1152,6 +1155,7 @@ export type QueryArticleArgs = {
export type QueryArticlesArgs = {
after?: InputMaybe<Scalars['String']>;
first?: InputMaybe<Scalars['Int']>;
includePending?: InputMaybe<Scalars['Boolean']>;
query?: InputMaybe<Scalars['String']>;
sharedOnly?: InputMaybe<Scalars['Boolean']>;
sort?: InputMaybe<SortParams>;
@ -1381,6 +1385,7 @@ export type SearchItem = {
readingProgressPercent?: Maybe<Scalars['Float']>;
shortId?: Maybe<Scalars['String']>;
slug: Scalars['String'];
state?: Maybe<ArticleSavingRequestStatus>;
subscription?: Maybe<Scalars['String']>;
title: Scalars['String'];
unsubHttpUrl?: Maybe<Scalars['String']>;
@ -2649,6 +2654,7 @@ export type ArticleResolvers<ContextType = ResolverContext, ParentType extends R
siteIcon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
siteName?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
slug?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
state?: Resolver<Maybe<ResolversTypes['ArticleSavingRequestStatus']>, ParentType, ContextType>;
subscription?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
unsubHttpUrl?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
@ -2678,6 +2684,7 @@ export type ArticleSavingRequestResolvers<ContextType = ResolverContext, ParentT
createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
errorCode?: Resolver<Maybe<ResolversTypes['CreateArticleErrorCode']>, ParentType, ContextType>;
id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
slug?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
status?: Resolver<ResolversTypes['ArticleSavingRequestStatus'], ParentType, ContextType>;
updatedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
user?: Resolver<ResolversTypes['User'], ParentType, ContextType>;
@ -3378,6 +3385,7 @@ export type SearchItemResolvers<ContextType = ResolverContext, ParentType extend
readingProgressPercent?: Resolver<Maybe<ResolversTypes['Float']>, ParentType, ContextType>;
shortId?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
slug?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
state?: Resolver<Maybe<ResolversTypes['ArticleSavingRequestStatus']>, ParentType, ContextType>;
subscription?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
unsubHttpUrl?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;

7
packages/api/src/generated/schema.graphql
View File

@ -50,6 +50,7 @@ type Article {
siteIcon: String
siteName: String
slug: String!
state: ArticleSavingRequestStatus
subscription: String
title: String!
unsubHttpUrl: String
@ -80,10 +81,11 @@ input ArticleHighlightsInput {
union ArticleResult = ArticleError | ArticleSuccess
type ArticleSavingRequest {
article: Article
article: Article @deprecated(reason: "article has been replaced with slug")
createdAt: Date!
errorCode: CreateArticleErrorCode
id: ID!
slug: String!
status: ArticleSavingRequestStatus!
updatedAt: Date!
user: User!
@ -817,7 +819,7 @@ type Profile {
type Query {
article(slug: String!, username: String!): ArticleResult!
articles(after: String, first: Int, query: String, sharedOnly: Boolean, sort: SortParams): ArticlesResult!
articles(after: String, first: Int, includePending: Boolean, query: String, sharedOnly: Boolean, sort: SortParams): ArticlesResult!
articleSavingRequest(id: ID!): ArticleSavingRequestResult!
feedArticles(after: String, first: Int, sharedByUser: ID, sort: SortParams): FeedArticlesResult!
getFollowers(userId: ID): GetFollowersResult!
@ -990,6 +992,7 @@ type SearchItem {
readingProgressPercent: Float
shortId: String
slug: String!
state: ArticleSavingRequestStatus
subscription: String
title: String!
unsubHttpUrl: String

135
packages/api/src/resolvers/article/index.ts
View File

@ -19,7 +19,6 @@ import {
MutationSetBookmarkArticleArgs,
MutationSetShareArticleArgs,
PageInfo,
PageType,
QueryArticleArgs,
QueryArticlesArgs,
QuerySearchArgs,
@ -45,10 +44,9 @@ import {
} from '../../utils/uploads'
import { ContentParseError } from '../../utils/errors'
import {
articleSavingRequestError,
articleSavingRequestPopulate,
authorized,
generateSlug,
pageError,
stringToHash,
userDataToUser,
validatedDate,
@ -72,7 +70,12 @@ import { createIntercomEvent } from '../../utils/intercom'
import { analytics } from '../../utils/analytics'
import { env } from '../../env'
import { Page, SearchItem as SearchItemData } from '../../elastic/types'
import {
ArticleSavingRequestStatus,
Page,
PageType,
SearchItem as SearchItemData,
} from '../../elastic/types'
import {
createPage,
deletePage,
@ -100,6 +103,7 @@ const FORCE_PUPPETEER_URLS = [
/twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/,
/^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/,
]
const UNPARSEABLE_CONTENT = 'We were unable to parse this page.'
export type CreateArticlesSuccessPartial = Merge<
CreateArticleSuccess,
@ -116,7 +120,7 @@ export const createArticleResolver = authorized<
input: {
url,
preparedDocument,
articleSavingRequestId,
articleSavingRequestId: pageId,
uploadFileId,
skipParsing,
source,
@ -142,25 +146,15 @@ export const createArticleResolver = authorized<
})
await createIntercomEvent('link-saved', uid)
const articleSavingRequest = articleSavingRequestId
? (await models.articleSavingRequest.get(articleSavingRequestId)) ||
(await authTrx((tx) =>
models.articleSavingRequest.create(
{ userId: uid, id: articleSavingRequestId },
tx
)
))
: undefined
const user = userDataToUser(await models.user.get(uid))
try {
if (isSiteBlockedForParse(url)) {
return articleSavingRequestError(
return pageError(
{
errorCodes: [CreateArticleErrorCode.NotAllowedToParse],
},
ctx,
articleSavingRequest
pageId
)
}
@ -213,10 +207,10 @@ export const createArticleResolver = authorized<
userId: uid,
})
if (!uploadFile) {
return articleSavingRequestError(
return pageError(
{ errorCodes: [CreateArticleErrorCode.UploadFileMissing] },
ctx,
articleSavingRequest
pageId
)
}
const uploadFileDetails = await getStorageFileDetails(
@ -281,14 +275,12 @@ export const createArticleResolver = authorized<
siteIcon: parsedContent?.siteIcon,
readingProgressPercent: 0,
readingProgressAnchorIndex: 0,
state: ArticleSavingRequestStatus.Succeeded,
}
let archive = false
if (articleSavingRequestId) {
const reminder = await models.reminder.getByRequestId(
uid,
articleSavingRequestId
)
if (pageId) {
const reminder = await models.reminder.getByRequestId(uid, pageId)
if (reminder) {
archive = reminder.archiveUntil || false
}
@ -313,12 +305,12 @@ export const createArticleResolver = authorized<
return await models.uploadFile.setFileUploadComplete(uploadFileId, tx)
})
if (!uploadFileData || !uploadFileData.id || !uploadFileData.fileName) {
return articleSavingRequestError(
return pageError(
{
errorCodes: [CreateArticleErrorCode.UploadFileMissing],
},
ctx,
articleSavingRequest
pageId
)
}
uploadFileUrlOverride = await makeStorageFilePublic(
@ -330,6 +322,7 @@ export const createArticleResolver = authorized<
const existingPage = await getPageByParam({
userId: uid,
url: articleToSave.url,
state: ArticleSavingRequestStatus.Succeeded,
})
if (existingPage) {
// update existing page in elastic
@ -345,17 +338,34 @@ export const createArticleResolver = authorized<
articleToSave = existingPage
} else {
// create new page in elastic
const pageId = await createPage(articleToSave, { ...ctx, uid })
if (!pageId) {
return articleSavingRequestError(
{
errorCodes: [CreateArticleErrorCode.ElasticError],
},
ctx,
articleSavingRequest
)
pageId = await createPage(articleToSave, { ...ctx, uid })
if (!pageId) {
return pageError(
{
errorCodes: [CreateArticleErrorCode.ElasticError],
},
ctx,
pageId
)
}
} else {
const updated = await updatePage(pageId, articleToSave, {
...ctx,
uid,
})
if (!updated) {
return pageError(
{
errorCodes: [CreateArticleErrorCode.ElasticError],
},
ctx,
pageId
)
}
}
log.info(
'page created in elastic',
pageId,
@ -370,25 +380,20 @@ export const createArticleResolver = authorized<
...articleToSave,
isArchived: !!articleToSave.archivedAt,
}
return articleSavingRequestPopulate(
{
user,
created: false,
createdArticle: createdArticle,
},
ctx,
articleSavingRequest?.id,
createdArticle.id || undefined
)
return {
user,
created: false,
createdArticle: createdArticle,
}
} catch (error) {
if (
error instanceof ContentParseError &&
error.message === 'UNABLE_TO_PARSE'
) {
return articleSavingRequestError(
return pageError(
{ errorCodes: [CreateArticleErrorCode.UnableToParse] },
ctx,
articleSavingRequest
pageId
)
}
throw error
@ -426,10 +431,19 @@ export const getArticleResolver: ResolverFn<
return { errorCodes: [ArticleErrorCode.NotFound] }
}
if (
page.state === ArticleSavingRequestStatus.Processing &&
new Date(page.createdAt).getTime() < new Date().getTime() - 1000 * 60
) {
page.content = UNPARSEABLE_CONTENT
page.description = UNPARSEABLE_CONTENT
}
return {
article: { ...page, isArchived: !!page.archivedAt, linkId: page.id },
}
} catch (error) {
console.log(error)
return { errorCodes: [ArticleErrorCode.BadData] }
}
}
@ -483,6 +497,7 @@ export const getArticlesResolver = authorized<
savedDateFilter: searchQuery.savedDateFilter,
publishedDateFilter: searchQuery.publishedDateFilter,
subscriptionFilter: searchQuery.subscriptionFilter,
includePending: params.includePending,
},
claims.uid
)) || [[], 0]
@ -618,29 +633,29 @@ export const setBookmarkArticleResolver = authorized<
{ input: { articleID, bookmark } },
{ models, authTrx, claims: { uid }, log, pubsub }
) => {
const article = await getPageById(articleID)
if (!article) {
const page = await getPageById(articleID)
if (!page) {
return { errorCodes: [SetBookmarkArticleErrorCode.NotFound] }
}
if (!bookmark) {
const userArticleRemoved = await getPageByParam({
const pageRemoved = await getPageByParam({
userId: uid,
_id: articleID,
})
if (!userArticleRemoved) {
if (!pageRemoved) {
return { errorCodes: [SetBookmarkArticleErrorCode.NotFound] }
}
await deletePage(userArticleRemoved.id, { pubsub, uid })
await deletePage(pageRemoved.id, { pubsub, uid })
const highlightsUnshared = await authTrx(async (tx) => {
return models.highlight.unshareAllHighlights(articleID, uid, tx)
})
log.info('Article unbookmarked', {
article: Object.assign({}, article, {
page: Object.assign({}, page, {
content: undefined,
originalHtml: undefined,
}),
@ -655,7 +670,7 @@ export const setBookmarkArticleResolver = authorized<
// Make sure article.id instead of userArticle.id has passed. We use it for cache updates
return {
bookmarkedArticle: {
...userArticleRemoved,
...pageRemoved,
isArchived: false,
savedByViewer: false,
postedByViewer: false,
@ -663,14 +678,14 @@ export const setBookmarkArticleResolver = authorized<
}
} else {
try {
const userArticle: Partial<Page> = {
const pageUpdated: Partial<Page> = {
userId: uid,
slug: generateSlug(article.title),
slug: generateSlug(page.title),
}
await updatePage(articleID, userArticle, { pubsub, uid })
await updatePage(articleID, pageUpdated, { pubsub, uid })
log.info('Article bookmarked', {
article: Object.assign({}, article, {
page: Object.assign({}, page, {
content: undefined,
originalHtml: undefined,
}),
@ -684,8 +699,8 @@ export const setBookmarkArticleResolver = authorized<
// Make sure article.id instead of userArticle.id has passed. We use it for cache updates
return {
bookmarkedArticle: {
...userArticle,
...article,
...pageUpdated,
...page,
isArchived: false,
savedByViewer: true,
postedByViewer: false,

50
packages/api/src/resolvers/article_saving_request/index.ts
View File

@ -1,29 +1,39 @@
/* eslint-disable prefer-const */
import {
ArticleSavingRequestSuccess,
ArticleSavingRequestError,
QueryArticleSavingRequestArgs,
ArticleSavingRequestErrorCode,
CreateArticleSavingRequestSuccess,
ArticleSavingRequestSuccess,
CreateArticleSavingRequestError,
CreateArticleSavingRequestErrorCode,
CreateArticleSavingRequestSuccess,
MutationCreateArticleSavingRequestArgs,
QueryArticleSavingRequestArgs,
} from '../../generated/graphql'
import {
authorized,
articleSavingRequestDataToArticleSavingRequest,
} from '../../utils/helpers'
import { authorized, pageToArticleSavingRequest } from '../../utils/helpers'
import { createPageSaveRequest } from '../../services/create_page_save_request'
import { createIntercomEvent } from '../../utils/intercom'
import { getPageById } from '../../elastic/pages'
import { isErrorWithCode } from '../user'
export const createArticleSavingRequestResolver = authorized<
CreateArticleSavingRequestSuccess,
CreateArticleSavingRequestError,
MutationCreateArticleSavingRequestArgs
>(async (_, { input: { url } }, { models, claims }) => {
>(async (_, { input: { url } }, { models, claims, pubsub }) => {
await createIntercomEvent('link-save-request', claims.uid)
const request = await createPageSaveRequest(claims.uid, url, models)
return {
articleSavingRequest: request,
try {
const request = await createPageSaveRequest(claims.uid, url, models, pubsub)
return {
articleSavingRequest: request,
}
} catch (err) {
console.log('error', err)
if (isErrorWithCode(err)) {
return {
errorCodes: [err.errorCode as CreateArticleSavingRequestErrorCode],
}
}
return { errorCodes: [CreateArticleSavingRequestErrorCode.BadData] }
}
})
@ -32,20 +42,18 @@ export const articleSavingRequestResolver = authorized<
ArticleSavingRequestError,
QueryArticleSavingRequestArgs
>(async (_, { id }, { models }) => {
let articleSavingRequest
let page
let user
try {
articleSavingRequest = await models.articleSavingRequest.get(id)
user = await models.user.get(articleSavingRequest.userId)
page = await getPageById(id)
if (!page) {
return { errorCodes: [ArticleSavingRequestErrorCode.NotFound] }
}
user = await models.user.get(page.userId)
// eslint-disable-next-line no-empty
} catch (error) {}
if (user && articleSavingRequest)
return {
articleSavingRequest: articleSavingRequestDataToArticleSavingRequest(
user,
articleSavingRequest
),
}
if (user && page)
return { articleSavingRequest: pageToArticleSavingRequest(user, page) }
return { errorCodes: [ArticleSavingRequestErrorCode.NotFound] }
})

6
packages/api/src/routers/article_router.ts
View File

@ -10,7 +10,6 @@ import { env } from './../env'
import { buildLogger } from './../utils/logger'
import * as jwt from 'jsonwebtoken'
import { corsConfig } from '../utils/corsConfig'
import { v4 as uuidv4 } from 'uuid'
import { createPageSaveRequest } from '../services/create_page_save_request'
import { initModels } from '../server'
import { kx } from '../datalayer/knex_config'
@ -45,9 +44,8 @@ export function articleRouter() {
return res.status(400).send({ errorCode: 'BAD_DATA' })
}
const requestId = uuidv4()
const models = initModels(kx, false)
const result = await createPageSaveRequest(uid, url, models, requestId)
const result = await createPageSaveRequest(uid, url, models)
if (isSiteBlockedForParse(url)) {
return res
@ -60,7 +58,7 @@ export function articleRouter() {
}
return res.send({
articleSavingRequestId: requestId,
articleSavingRequestId: result.id,
})
})
return router

3
packages/api/src/routers/svc/pdf_attachments.ts
View File

@ -15,7 +15,7 @@ import { getNewsletterEmail } from '../../services/newsletters'
import { setClaims } from '../../datalayer/helpers'
import { generateSlug } from '../../utils/helpers'
import { createPubSubClient } from '../../datalayer/pubsub'
import { Page } from '../../elastic/types'
import { ArticleSavingRequestStatus, Page } from '../../elastic/types'
import { createPage } from '../../elastic/pages'
export function pdfAttachmentsRouter() {
@ -157,6 +157,7 @@ export function pdfAttachmentsRouter() {
createdAt: new Date(),
readingProgressPercent: 0,
readingProgressAnchorIndex: 0,
state: ArticleSavingRequestStatus.Succeeded,
}
const pageId = await createPage(articleToSave, {

6
packages/api/src/schema.ts
View File

@ -345,6 +345,7 @@ const schema = gql`
subscription: String
unsubMailTo: String
unsubHttpUrl: String
state: ArticleSavingRequestStatus
}
# Query: article
@ -954,7 +955,8 @@ const schema = gql`
id: ID!
userId: ID! @deprecated(reason: "userId has been replaced with user")
user: User!
article: Article
article: Article @deprecated(reason: "article has been replaced with slug")
slug: String!
status: ArticleSavingRequestStatus!
errorCode: CreateArticleErrorCode
createdAt: Date!
@ -1435,6 +1437,7 @@ const schema = gql`
subscription: String
unsubMailTo: String
unsubHttpUrl: String
state: ArticleSavingRequestStatus
}
type SearchItemEdge {
@ -1583,6 +1586,7 @@ const schema = gql`
after: String
first: Int
query: String
includePending: Boolean
): ArticlesResult!
article(username: String!, slug: String!): ArticleResult!
sharedArticle(

57
packages/api/src/services/create_page_save_request.ts
View File

@ -7,9 +7,14 @@ import {
ArticleSavingRequest,
CreateArticleSavingRequestErrorCode,
} from '../generated/graphql'
import { articleSavingRequestDataToArticleSavingRequest } from '../utils/helpers'
import { generateSlug, pageToArticleSavingRequest } from '../utils/helpers'
import * as privateIpLib from 'private-ip'
import { countByCreatedAt } from '../elastic/pages'
import { countByCreatedAt, createPage, getPageByParam } from '../elastic/pages'
import { ArticleSavingRequestStatus, Page, PageType } from '../elastic/types'
import { createPubSubClient, PubsubClient } from '../datalayer/pubsub'
import normalizeUrl from 'normalize-url'
const SAVING_DESCRIPTION = 'Your link is being saved...'
const isPrivateIP = privateIpLib.default
@ -53,6 +58,7 @@ export const createPageSaveRequest = async (
userId: string,
url: string,
models: DataModels,
pubsub: PubsubClient = createPubSubClient(),
articleSavingRequestId = uuidv4(),
priority?: 'low' | 'high'
): Promise<ArticleSavingRequest> => {
@ -76,6 +82,10 @@ export const createPageSaveRequest = async (
// get priority by checking rate limit if not specified
priority = priority || (await getPriorityByRateLimit(userId))
url = normalizeUrl(url, {
stripHash: true,
stripWWW: false,
})
const createdTaskName = await enqueueParseRequest(
url,
userId,
@ -83,14 +93,41 @@ export const createPageSaveRequest = async (
priority
)
const articleSavingRequestData = await models.articleSavingRequest.create({
userId: userId,
taskName: createdTaskName,
id: articleSavingRequestId,
const existingPage = await getPageByParam({
userId,
url,
state: ArticleSavingRequestStatus.Succeeded,
})
if (existingPage) {
console.log('Page already exists', url)
existingPage.taskName = createdTaskName
return pageToArticleSavingRequest(user, existingPage)
}
return articleSavingRequestDataToArticleSavingRequest(
user,
articleSavingRequestData
)
const page: Page = {
id: articleSavingRequestId,
userId,
content: SAVING_DESCRIPTION,
createdAt: new Date(),
hash: '',
pageType: PageType.Unknown,
readingProgressAnchorIndex: 0,
readingProgressPercent: 0,
slug: generateSlug(url),
title: url,
url,
taskName: createdTaskName,
state: ArticleSavingRequestStatus.Processing,
description: SAVING_DESCRIPTION,
}
const pageId = await createPage(page, { pubsub, uid: userId })
if (!pageId) {
console.log('Failed to create page', page)
return Promise.reject({
errorCode: CreateArticleSavingRequestErrorCode.BadData,
})
}
return pageToArticleSavingRequest(user, page)
}

3
packages/api/src/services/save_email.ts
View File

@ -6,7 +6,7 @@ import {
} from '../utils/parser'
import normalizeUrl from 'normalize-url'
import { PubsubClient } from '../datalayer/pubsub'
import { Page } from '../elastic/types'
import { ArticleSavingRequestStatus, Page } from '../elastic/types'
import { createPage, getPageByParam, updatePage } from '../elastic/pages'
export type SaveContext = {
@ -69,6 +69,7 @@ export const saveEmail = async (
readingProgressAnchorIndex: 0,
readingProgressPercent: 0,
subscription: input.author,
state: ArticleSavingRequestStatus.Succeeded,
}
const page = await getPageByParam({ userId: ctx.uid, url: articleToSave.url })

33
packages/api/src/services/save_file.ts
View File

@ -3,7 +3,6 @@ import { PubsubClient } from '../datalayer/pubsub'
import { UserData } from '../datalayer/user/model'
import { homePageURL } from '../env'
import {
ArticleSavingRequestStatus,
PageType,
SaveErrorCode,
SaveFileInput,
@ -12,8 +11,8 @@ import {
import { DataModels } from '../resolvers/types'
import { generateSlug } from '../utils/helpers'
import { getStorageFileDetails, makeStorageFilePublic } from '../utils/uploads'
import { createSavingRequest } from './save_page'
import { createPage, getPageByParam, updatePage } from '../elastic/pages'
import { ArticleSavingRequestStatus } from '../elastic/types'
type SaveContext = {
pubsub: PubsubClient
@ -45,8 +44,6 @@ export const saveFile = async (
}
}
const savingRequest = await createSavingRequest(ctx, input.clientRequestId)
const uploadFileDetails = await getStorageFileDetails(
input.uploadFileId,
uploadFile.fileName
@ -71,6 +68,7 @@ export const saveFile = async (
const matchedUserArticleRecord = await getPageByParam({
userId: saver.id,
url: uploadFileUrlOverride,
state: ArticleSavingRequestStatus.Succeeded,
})
if (matchedUserArticleRecord) {
@ -82,17 +80,7 @@ export const saveFile = async (
},
ctx
)
await ctx.authTrx(async (tx) => {
await ctx.models.articleSavingRequest.update(
savingRequest.id,
{
elasticPageId: matchedUserArticleRecord.id,
status: ArticleSavingRequestStatus.Succeeded,
},
tx
)
})
input.clientRequestId = matchedUserArticleRecord.id
} else {
const pageId = await createPage(
{
@ -104,10 +92,11 @@ export const saveFile = async (
uploadFileId: input.uploadFileId,
slug: generateSlug(uploadFile.fileName),
userId: saver.id,
id: '',
id: input.clientRequestId,
createdAt: new Date(),
readingProgressPercent: 0,
readingProgressAnchorIndex: 0,
state: ArticleSavingRequestStatus.Succeeded,
},
ctx
)
@ -118,17 +107,7 @@ export const saveFile = async (
errorCodes: [SaveErrorCode.Unknown],
}
}
await ctx.authTrx(async (tx) => {
await ctx.models.articleSavingRequest.update(
savingRequest.id,
{
elasticPageId: pageId,
status: ArticleSavingRequestStatus.Succeeded,
},
tx
)
})
input.clientRequestId = pageId
}
return {

51
packages/api/src/services/save_page.ts
View File

@ -1,20 +1,13 @@
import { PubsubClient } from '../datalayer/pubsub'
import { homePageURL } from '../env'
import {
ArticleSavingRequestStatus,
Maybe,
SavePageInput,
SaveResult,
} from '../generated/graphql'
import { Maybe, SavePageInput, SaveResult } from '../generated/graphql'
import { DataModels } from '../resolvers/types'
import { generateSlug, stringToHash, validatedDate } from '../utils/helpers'
import { parseOriginalContent, parsePreparedContent } from '../utils/parser'
import normalizeUrl from 'normalize-url'
import { createPageSaveRequest } from './create_page_save_request'
import { kx } from '../datalayer/knex_config'
import { setClaims } from '../datalayer/helpers'
import { Page } from '../elastic/types'
import { ArticleSavingRequestStatus, Page } from '../elastic/types'
import { createPage, getPageByParam, updatePage } from '../elastic/pages'
type SaveContext = {
@ -70,8 +63,6 @@ export const savePage = async (
saver: SaverUserData,
input: SavePageInput
): Promise<SaveResult> => {
const savingRequest = await createSavingRequest(ctx, input.clientRequestId)
const [slug, croppedPathname] = createSlug(input.url, input.title)
const parseResult = await parsePreparedContent(input.url, {
document: input.originalContent,
@ -84,7 +75,7 @@ export const savePage = async (
const pageType = parseOriginalContent(input.url, input.originalContent)
const articleToSave: Page = {
id: '',
id: input.clientRequestId,
slug,
userId: saver.userId,
originalHtml: parseResult.domContent,
@ -103,11 +94,13 @@ export const savePage = async (
createdAt: new Date(),
readingProgressPercent: 0,
readingProgressAnchorIndex: 0,
state: ArticleSavingRequestStatus.Succeeded,
}
const existingPage = await getPageByParam({
userId: saver.userId,
url: articleToSave.url,
state: ArticleSavingRequestStatus.Succeeded,
})
if (existingPage) {
await updatePage(
@ -118,33 +111,17 @@ export const savePage = async (
},
ctx
)
await kx.transaction(async (tx) => {
await setClaims(tx, saver.userId)
await ctx.models.articleSavingRequest.update(
savingRequest.id,
{
status: ArticleSavingRequestStatus.Succeeded,
elasticPageId: existingPage.id,
},
tx
)
})
input.clientRequestId = existingPage.id
} else if (shouldParseInBackend(input)) {
await createPageSaveRequest(saver.userId, input.url, ctx.models)
await createPageSaveRequest(
saver.userId,
input.url,
ctx.models,
ctx.pubsub,
input.clientRequestId
)
} else {
const pageId = await createPage(articleToSave, ctx)
await kx.transaction(async (tx) => {
await setClaims(tx, saver.userId)
await ctx.models.articleSavingRequest.update(
savingRequest.id,
{
status: ArticleSavingRequestStatus.Succeeded,
elasticPageId: pageId,
},
tx
)
})
await createPage(articleToSave, ctx)
}
return {

22
packages/api/src/services/save_url.ts
View File

@ -1,7 +1,7 @@
import { PubsubClient } from '../datalayer/pubsub'
import { UserData } from '../datalayer/user/model'
import { homePageURL } from '../env'
import { SaveResult, SaveUrlInput } from '../generated/graphql'
import { SaveErrorCode, SaveResult, SaveUrlInput } from '../generated/graphql'
import { DataModels } from '../resolvers/types'
import { createPageSaveRequest } from './create_page_save_request'
@ -16,20 +16,24 @@ export const saveUrl = async (
input: SaveUrlInput
): Promise<SaveResult> => {
try {
await createPageSaveRequest(
const pageSaveRequest = await createPageSaveRequest(
saver.id,
input.url,
ctx.models,
ctx.pubsub,
input.clientRequestId
)
return {
clientRequestId: pageSaveRequest.id,
url: `${homePageURL()}/${saver.profile.username}/links/${
pageSaveRequest.id
}`,
}
} catch (error) {
console.log('error enqueuing request', error)
}
return {
clientRequestId: input.clientRequestId,
url: `${homePageURL()}/${saver.profile.username}/links/${
input.clientRequestId
}`,
return {
errorCodes: [SaveErrorCode.Unknown],
}
}
}

57
packages/api/src/utils/helpers.ts
View File

@ -1,9 +1,7 @@
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import {
ArticleSavingRequest,
ArticleSavingRequestStatus,
CreateArticleError,
CreateArticleErrorCode,
FeedArticle,
Profile,
ResolverFn,
@ -17,8 +15,9 @@ import {
import crypto from 'crypto'
import slugify from 'voca/slugify'
import { Merge } from '../util'
import { ArticleSavingRequestData } from '../datalayer/article_saving_request/model'
import { CreateArticlesSuccessPartial } from '../resolvers'
import { ArticleSavingRequestStatus, Page } from '../elastic/types'
import { updatePage } from '../elastic/pages'
interface InputObject {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@ -175,56 +174,32 @@ export const generateSlug = (title: string): string => {
export const MAX_CONTENT_LENGTH = 5e7 //50MB
export const articleSavingRequestError = async (
export const pageError = async (
result: CreateArticleError,
ctx: WithDataSourcesContext,
articleSavingReqest?: ArticleSavingRequestData
pageId?: string | null
): Promise<CreateArticleError | CreateArticlesSuccessPartial> => {
if (!articleSavingReqest) return result
if (!pageId) return result
await ctx.authTrx((tx) =>
ctx.models.articleSavingRequest.update(
articleSavingReqest.id,
{
status: ArticleSavingRequestStatus.Failed,
errorCode: result.errorCodes[0],
},
tx
)
await updatePage(
pageId,
{
state: ArticleSavingRequestStatus.Failed,
},
ctx
)
return result
}
export const articleSavingRequestPopulate = async (
result: CreateArticlesSuccessPartial,
ctx: WithDataSourcesContext,
articleSavingReqestId: string | undefined,
articleId: string | undefined
): Promise<CreateArticleError | CreateArticlesSuccessPartial> => {
if (!articleSavingReqestId) return result
await ctx.authTrx((tx) =>
ctx.models.articleSavingRequest.update(
articleSavingReqestId,
{
status: ArticleSavingRequestStatus.Succeeded,
elasticPageId: articleId,
},
tx
)
)
return result
}
export const articleSavingRequestDataToArticleSavingRequest = (
export const pageToArticleSavingRequest = (
user: UserData,
articleSavingRequest: ArticleSavingRequestData
page: Page
): ArticleSavingRequest => ({
...articleSavingRequest,
...page,
user: userDataToUser(user),
status: articleSavingRequest.status as ArticleSavingRequestStatus,
errorCode: articleSavingRequest.errorCode as CreateArticleErrorCode,
status: page.state,
updatedAt: page.updatedAt || new Date(),
})
export const validatedDate = (

11
packages/api/test/elastic/index.test.ts
View File

@ -1,7 +1,13 @@
import 'mocha'
import { expect } from 'chai'
import { InFilter, ReadFilter } from '../../src/utils/search'
import { Highlight, Page, PageContext, PageType } from '../../src/elastic/types'
import {
ArticleSavingRequestStatus,
Highlight,
Page,
PageContext,
PageType,
} from '../../src/elastic/types'
import { createPubSubClient } from '../../src/datalayer/pubsub'
import {
countByCreatedAt,
@ -58,6 +64,7 @@ describe('elastic api', () => {
createdAt: new Date(),
},
],
state: ArticleSavingRequestStatus.Succeeded,
}
const pageId = await createPage(page, ctx)
if (!pageId) {
@ -94,6 +101,7 @@ describe('elastic api', () => {
readingProgressPercent: 0,
readingProgressAnchorIndex: 0,
url: 'https://blog.omnivore.app/testUrl',
state: ArticleSavingRequestStatus.Succeeded,
}
newPageId = await createPage(newPageData, ctx)
@ -197,6 +205,7 @@ describe('elastic api', () => {
readingProgressPercent: 0,
readingProgressAnchorIndex: 0,
url: 'https://blog.omnivore.app/testCount',
state: ArticleSavingRequestStatus.Succeeded,
}
await createPage(newPageData, ctx)

117
packages/api/test/resolvers/article.test.ts
View File

@ -12,7 +12,13 @@ import { User } from '../../src/entity/user'
import chaiString from 'chai-string'
import { Label } from '../../src/entity/label'
import { UploadFileStatus } from '../../src/generated/graphql'
import { Highlight, Page, PageContext, PageType } from '../../src/elastic/types'
import {
ArticleSavingRequestStatus,
Highlight,
Page,
PageContext,
PageType,
} from '../../src/elastic/types'
import { UploadFile } from '../../src/entity/upload_file'
import { createPubSubClient } from '../../src/datalayer/pubsub'
import { getRepository } from '../../src/entity/utils'
@ -133,6 +139,7 @@ const getArticleQuery = (slug: string) => {
article {
id
slug
content
highlights {
id
shortId
@ -230,6 +237,27 @@ const saveFileQuery = (url: string, uploadFileId: string) => {
`
}
const saveUrlQuery = (url: string) => {
return `
mutation {
saveUrl(
input: {
url: "${url}",
source: "test",
clientRequestId: "${generateFakeUuid()}",
}
) {
... on SaveSuccess {
url
}
... on SaveError {
errorCodes
}
}
}
`
}
const setBookmarkQuery = (articleId: string, bookmark: boolean) => {
return `
mutation {
@ -344,7 +372,7 @@ describe('Article API', () => {
let query = ''
let slug = ''
let pageId: string | undefined
let pageId: string
before(async () => {
const page = {
@ -371,13 +399,12 @@ describe('Article API', () => {
},
],
} as Page
pageId = await createPage(page, ctx)
const id = await createPage(page, ctx)
id && (pageId = id)
})
after(async () => {
if (pageId) {
await deletePage(pageId, ctx)
}
await deletePage(pageId, ctx)
})
beforeEach(async () => {
@ -400,6 +427,27 @@ describe('Article API', () => {
expect(res.body.data.article.article.highlights).to.length(1)
})
context('when page is failed to process', () => {
before(async () => {
await updatePage(
pageId,
{
state: ArticleSavingRequestStatus.Processing,
createdAt: new Date(Date.now() - 1000 * 60),
},
ctx
)
})
it('should return unable to parse', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.article.article.content).to.eql(
'We were unable to parse this page.'
)
})
})
})
context('when page does not exist', () => {
@ -613,6 +661,61 @@ describe('Article API', () => {
})
})
describe('SaveUrl', () => {
let query = ''
let url = 'https://example.com/new-url-1'
beforeEach(() => {
query = saveUrlQuery(url)
})
context('when we save a new url', () => {
it('should return a slugged url', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.saveUrl.url).to.startsWith(
'http://localhost:3000/fakeUser/links/'
)
})
})
context('when we save a url that is already archived', () => {
it('it should return that page in the GetArticles Query', async () => {
url = 'https://example.com/new-url'
await graphqlRequest(saveUrlQuery(url), authToken).expect(200)
let allLinks
// Save a link, then archive it
// set a slight delay to make sure the page is updated
setTimeout(async () => {
let allLinks = await graphqlRequest(
articlesQuery(''),
authToken
).expect(200)
const justSavedId = allLinks.body.data.articles.edges[0].node.id
await archiveLink(authToken, justSavedId)
}, 100)
// test the negative case, ensuring the archive link wasn't returned
setTimeout(async () => {
allLinks = await graphqlRequest(articlesQuery(''), authToken).expect(
200
)
expect(allLinks.body.data.articles.edges[0].node.url).to.not.eq(url)
}, 100)
// Now save the link again, and ensure it is returned
await graphqlRequest(saveUrlQuery(url), authToken).expect(200)
setTimeout(async () => {
allLinks = await graphqlRequest(articlesQuery(''), authToken).expect(
200
)
expect(allLinks.body.data.articles.edges[0].node.url).to.eq(url)
}, 100)
})
})
})
describe('setBookmarkArticle', () => {
let query = ''
let articleId = ''
@ -632,6 +735,7 @@ describe('Article API', () => {
slug: 'test-with-omnivore',
readingProgressPercent: 0,
readingProgressAnchorIndex: 0,
state: ArticleSavingRequestStatus.Succeeded,
}
const newPageId = await createPage(page, ctx)
if (newPageId) {
@ -803,6 +907,7 @@ describe('Article API', () => {
readingProgressAnchorIndex: 0,
url: url,
savedAt: new Date(),
state: ArticleSavingRequestStatus.Succeeded,
}
const pageId = await createPage(page, ctx)
if (!pageId) {

149
packages/api/test/resolvers/article_saving_request.test.ts Normal file
View File

@ -0,0 +1,149 @@
import { User } from '../../src/entity/user'
import {
ArticleSavingRequestStatus,
PageContext,
} from '../../src/elastic/types'
import { createTestUser, deleteTestUser } from '../db'
import { graphqlRequest, request } from '../util'
import { createPubSubClient } from '../../src/datalayer/pubsub'
import { expect } from 'chai'
import { describe } from 'mocha'
import { getPageById } from '../../src/elastic/pages'
import {
ArticleSavingRequestErrorCode,
CreateArticleSavingRequestErrorCode,
} from '../../src/generated/graphql'
const articleSavingRequestQuery = (id: string) => `
query {
articleSavingRequest(id: "${id}") {
... on ArticleSavingRequestSuccess {
articleSavingRequest {
id
status
}
}
... on ArticleSavingRequestError {
errorCodes
}
}
}
`
const createArticleSavingRequestMutation = (url: string) => `
mutation {
createArticleSavingRequest(input: {
url: "${url}"
}) {
... on CreateArticleSavingRequestSuccess {
articleSavingRequest {
id
status
}
}
... on CreateArticleSavingRequestError {
errorCodes
}
}
}
`
describe('ArticleSavingRequest API', () => {
const username = 'fakeUser'
let authToken: string
let user: User
let ctx: PageContext
before(async () => {
// create test user and login
user = await createTestUser(username)
const res = await request
.post('/local/debug/fake-user-login')
.send({ fakeEmail: user.email })
authToken = res.body.authToken
ctx = {
pubsub: createPubSubClient(),
refresh: true,
uid: user.id,
}
})
after(async () => {
// clean up
await deleteTestUser(username)
})
describe('createArticleSavingRequest', () => {
it('returns the article saving request', async () => {
const res = await graphqlRequest(
createArticleSavingRequestMutation('https://example.com'),
authToken
).expect(200)
expect(
res.body.data.createArticleSavingRequest.articleSavingRequest.status
).to.eql(ArticleSavingRequestStatus.Processing)
})
it('creates a page in elastic', async () => {
const res = await graphqlRequest(
createArticleSavingRequestMutation('https://example.com/1'),
authToken
).expect(200)
const page = await getPageById(
res.body.data.createArticleSavingRequest.articleSavingRequest.id
)
expect(page?.description).to.eq('Your link is being saved...')
})
it('returns an error if the url is invalid', async () => {
const res = await graphqlRequest(
createArticleSavingRequestMutation('invalid url'),
authToken
).expect(200)
expect(res.body.data.createArticleSavingRequest.errorCodes).to.eql([
CreateArticleSavingRequestErrorCode.BadData,
])
})
})
describe('articleSavingRequest', () => {
let articleSavingRequestId: string
before(async () => {
// create article saving request
const res = await graphqlRequest(
createArticleSavingRequestMutation('https://example.com/2'),
authToken
).expect(200)
articleSavingRequestId =
res.body.data.createArticleSavingRequest.articleSavingRequest.id
})
it('returns the article saving request if exists', async () => {
const res = await graphqlRequest(
articleSavingRequestQuery(articleSavingRequestId),
authToken
).expect(200)
expect(res.body.data.articleSavingRequest.articleSavingRequest.id).to.eql(
articleSavingRequestId
)
})
it('returns not_found if not exists', async () => {
const res = await graphqlRequest(
articleSavingRequestQuery('invalid-id'),
authToken
).expect(200)
expect(res.body.data.articleSavingRequest.errorCodes).to.eql([
ArticleSavingRequestErrorCode.NotFound,
])
})
})
})

3
packages/api/test/util.ts
View File

@ -2,7 +2,7 @@ import { createApp } from '../src/server'
import supertest from 'supertest'
import { v4 } from 'uuid'
import { corsConfig } from '../src/utils/corsConfig'
import { Page } from '../src/elastic/types'
import { ArticleSavingRequestStatus, Page } from '../src/elastic/types'
import { PageType } from '../src/generated/graphql'
import { User } from '../src/entity/user'
import { Label } from '../src/entity/label'
@ -56,6 +56,7 @@ export const createTestElasticPage = async (
labels: labels,
readingProgressPercent: 0,
readingProgressAnchorIndex: 0,
state: ArticleSavingRequestStatus.Succeeded,
}
const pageId = await createPage(page, {

1
packages/db/Dockerfile
View File

@ -14,5 +14,6 @@ RUN yarn install
ADD /packages/db ./packages/db
ADD /packages/db/setup.sh ./packages/db/setup.sh
ADD /packages/db/elastic_migrations ./packages/db/elastic_migrations
CMD ["yarn", "workspace", "@omnivore/db", "migrate"]

0
packages/api/add_highlight_to_elastic.py → packages/db/elastic_migrations/add_highlight_to_elastic.py
View File

0
packages/api/elastic_migrate.py → packages/db/elastic_migrations/elastic_migrate.py
View File

6
packages/api/index_settings.json → packages/db/elastic_migrations/index_settings.json
View File

@ -101,6 +101,12 @@
"subscription": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"state": {
"type": "keyword"
},
"taskName": {
"type": "keyword"
}
}
}

108
packages/db/migrate.ts
View File

@ -1,19 +1,22 @@
/* eslint-disable @typescript-eslint/naming-convention */
import * as dotenv from 'dotenv';
dotenv.config();
import * as dotenv from 'dotenv'
import Postgrator from 'postgrator'
import chalk from 'chalk'
import { Client } from '@elastic/elasticsearch'
import { readFileSync } from 'fs'
import { join } from 'path'
import Postgrator from 'postgrator';
import chalk from 'chalk';
dotenv.config()
const log = (text: string, style: typeof chalk.white = chalk.white): void =>
console.log(`${chalk.cyanBright('>')} ${style(text)}`);
console.log(`${chalk.cyanBright('>')} ${style(text)}`)
interface DBEnv {
host: string;
port: number;
database: string;
username: string;
password: string;
host: string
port: number
database: string
username: string
password: string
}
const getEnv = (): DBEnv => {
@ -23,14 +26,14 @@ const getEnv = (): DBEnv => {
PG_DB: database,
PG_USER: username,
PG_PASSWORD: password,
} = process.env;
} = process.env
if (typeof username !== 'string') {
throw new Error('No PG user passed in env');
throw new Error('No PG user passed in env')
}
if (typeof password !== 'string') {
throw new Error('No PG password passed in env');
throw new Error('No PG password passed in env')
}
const config = {
@ -39,10 +42,10 @@ const getEnv = (): DBEnv => {
database: database || 'omnivore',
username,
password,
};
}
return config;
};
return config
}
const postgrator = new Postgrator({
migrationDirectory: __dirname + '/migrations',
@ -52,37 +55,76 @@ const postgrator = new Postgrator({
schemaTable: 'schemaversion',
// Validate migration md5 checksum to ensure the contents of the script have not changed
validateChecksums: true,
});
})
log('Starting migration manager');
log('Starting migration manager')
const targetMigration = process.argv[2];
const targetMigration = process.argv[2]
const targetMigrationLabel = targetMigration
? `'${chalk.blue(targetMigration)}'`
: chalk.blue('latest');
: chalk.blue('latest')
log(`Migrating to ${targetMigrationLabel}.\n`);
log(`Migrating to ${targetMigrationLabel}.\n`)
const logAppliedMigrations = (appliedMigrations: Postgrator.Migration[]): void => {
const logAppliedMigrations = (
appliedMigrations: Postgrator.Migration[]
): void => {
if (appliedMigrations.length > 0) {
log(`Applied ${chalk.green(appliedMigrations.length.toString())} migrations successfully:`);
log(
`Applied ${chalk.green(
appliedMigrations.length.toString()
)} migrations successfully:`
)
for (const migration of appliedMigrations) {
const actionLabel = migration.action === 'do' ? chalk.green('+') : chalk.red('-');
console.log(` ${actionLabel} ${migration.name}`);
const actionLabel =
migration.action === 'do' ? chalk.green('+') : chalk.red('-')
console.log(` ${actionLabel} ${migration.name}`)
}
} else {
log(`No migrations applied.`);
log(`No migrations applied.`)
}
};
}
export const INDEX_ALIAS = 'pages_alias'
export const client = new Client({
node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200',
auth: {
username: process.env.ELASTICSEARCH_USERNAME || '',
password: process.env.ELASTICSEARCH_PASSWORD || '',
},
})
const updateMappings = async (): Promise<void> => {
// read index settings from file
const indexSettings = readFileSync(
join(__dirname, 'elastic_migrations', 'index_settings.json'),
'utf8'
)
// update mappings
await client.indices.putMapping({
index: INDEX_ALIAS,
body: JSON.parse(indexSettings).mappings,
})
}
postgrator
.migrate(targetMigration)
.then(logAppliedMigrations)
.catch(error => {
log(`${chalk.red('Migration failed: ')}${error.message}`, chalk.red);
const { appliedMigrations } = error;
logAppliedMigrations(appliedMigrations);
process.exit(1);
.catch((error) => {
log(`${chalk.red('Migration failed: ')}${error.message}`, chalk.red)
const { appliedMigrations } = error
logAppliedMigrations(appliedMigrations)
process.exit(1)
})
.then(() => console.log('\nExiting...'))
log('Starting updating elasticsearch index mappings...')
updateMappings()
.then(() => console.log('\nUpdating elastic completed.'))
.catch((error) => {
log(`${chalk.red('Updating failed: ')}${error.message}`, chalk.red)
process.exit(1)
})
.then(() => console.log('\nExiting...'));

1
packages/db/package.json
View File

@ -9,6 +9,7 @@
"author": "",
"license": "ISC",
"dependencies": {
"@elastic/elasticsearch": "~7.12.0",
"dotenv": "^8.2.0",
"pg": "^8.3.0",
"postgrator": "^4.1.1",

32
packages/web/components/templates/SavingRequest.tsx
View File

@ -5,11 +5,11 @@ import { StyledText } from '../elements/StyledText'
export function Loader(): JSX.Element {
const breathe = keyframes({
'0%': { transform: 'scale(0.8)' },
'25%': { transform: 'scale(1)' },
'50%': { transform: 'scale(0.8)' },
'75%': { transform: 'scale(1)' },
'100%': { transform: 'scale(0.8)' },
'0%': { content: '' },
'25%': { content: '.' },
'50%': { content: '..' },
'75%': { content: '...' },
'100%': { content: '....' },
})
return (
@ -18,22 +18,25 @@ export function Loader(): JSX.Element {
distribution="center"
css={{
pt: '$6',
bg: '$grayBase',
width: '100%',
}}
>
<Box
css={{
transform: 'scale(0.8)',
animation: `${breathe} 4s ease-out infinite`,
}}
>
<Box>
<LogoIcon
size={140}
strokeColor={theme.colors.grayTextContrast.toString()}
strokeColor={theme.colors.grayText.toString()}
/>
</Box>
<StyledText style="subHeadline">Saving Link...</StyledText>
<StyledText style="subHeadline" className='loading'
css={{
color: theme.colors.grayText.toString(),
'&:after': {
width: '10px',
display: 'inline-block',
content: 'test',
animation: `${breathe} steps(1,end) 2s infinite`,
},
}}>Saving Link</StyledText>
</VStack>
)
}
@ -55,7 +58,6 @@ export function ErrorComponent(props: ErrorComponentProps): JSX.Element {
css={{
pt: '$6',
px: '$3',
bg: '$grayBase',
width: '100%',
}}
>

45
packages/web/components/templates/article/ArticleActionsMenu.tsx
View File

@ -12,7 +12,7 @@ import { ReaderSettingsControl } from "./ReaderSettingsControl"
export type ArticleActionsMenuLayout = 'top' | 'side'
type ArticleActionsMenuProps = {
article: ArticleAttributes
article?: ArticleAttributes
layout: ArticleActionsMenuLayout
lineHeight: number
marginWidth: number
@ -97,24 +97,35 @@ export function ArticleActionsMenu(props: ArticleActionsMenuProps): JSX.Element
display: 'none',
}}}
>
<ActionDropdown
layout={props.layout}
triggerElement={
<TooltipWrapped
tooltipContent="Edit labels"
tooltipSide={props.layout == 'side' ? 'right' : 'bottom'}
{props.article ? (
<ActionDropdown
layout={props.layout}
triggerElement={
<TooltipWrapped
tooltipContent="Edit labels"
tooltipSide={props.layout == 'side' ? 'right' : 'bottom'}
>
<TagSimple size={24} color={theme.colors.readerFont.toString()} />
</TooltipWrapped>
}
>
<SetLabelsControl
article={props.article}
linkId={props.article.linkId}
labels={props.article.labels}
articleActionHandler={props.articleActionHandler}
/>
</ActionDropdown>
) : (
<Button style='articleActionIcon'
css={{
'@smDown': {
display: 'flex',
},
}}>
<TagSimple size={24} color={theme.colors.readerFont.toString()} />
</TooltipWrapped>
}
>
<SetLabelsControl
article={props.article}
linkId={props.article.linkId}
labels={props.article.labels}
articleActionHandler={props.articleActionHandler}
/>
</ActionDropdown>
</Button>
)}
</SpanBox>
<Button style='articleActionIcon'

69
packages/web/components/templates/article/SkeletonArticleContainer.tsx Normal file
View File

@ -0,0 +1,69 @@
import { Box } from '../../elements/LayoutPrimitives'
import { StyledText } from '../../elements/StyledText'
import { theme } from '../../tokens/stitches.config'
type SkeletonArticleContainerProps = {
margin?: number
fontSize?: number
fontFamily?: string
lineHeight?: number
children?: React.ReactNode
}
export function SkeletonArticleContainer(props: SkeletonArticleContainerProps): JSX.Element {
const styles = {
margin: props.margin ?? 360,
fontSize: props.fontSize ?? 20,
lineHeight: props.lineHeight ?? 150,
fontFamily: props.fontFamily ?? 'inter',
readerFontColor: theme.colors.readerFont.toString(),
readerFontColorTransparent: theme.colors.readerFontTransparent.toString(),
readerTableHeaderColor: theme.colors.readerTableHeader.toString(),
readerHeadersColor: theme.colors.readerHeader.toString(),
}
return (
<>
<Box
id="article-container"
css={{
width: '100%',
height: '100vh',
padding: '16px',
maxWidth: '100%',
background: theme.colors.grayBg.toString(),
'--text-font-family': styles.fontFamily,
'--text-font-size': `${styles.fontSize}px`,
'--line-height': `${styles.lineHeight}%`,
'--blockquote-padding': '0.5em 1em',
'--blockquote-icon-font-size': '1.3rem',
'--figure-margin': '1.6rem auto',
'--hr-margin': '1em',
'--font-color': styles.readerFontColor,
'--font-color-transparent': styles.readerFontColorTransparent,
'--table-header-color': styles.readerTableHeaderColor,
'--headers-color': styles.readerHeadersColor,
'@sm': {
'--blockquote-padding': '1em 2em',
'--blockquote-icon-font-size': '1.7rem',
'--figure-margin': '2.6875rem auto',
'--hr-margin': '2em',
margin: `30px 0px`,
},
'@md': {
maxWidth: 1024 - (styles.margin),
},
'@lg': {
margin: `30px 0`,
maxWidth: 1024 - (styles.margin),
},
}}
>
{props.children}
{/* <Box css={{ width: '100%', height: '100%', textAlign: 'center', verticalAlign: 'middle' }}>
Saving Page
</Box> */}
</Box>
</>
)
}

7
packages/web/components/templates/homeFeed/HomeFeedContainer.tsx
View File

@ -40,6 +40,7 @@ import { Label } from '../../../lib/networking/fragments/labelFragment'
import { isVipUser } from '../../../lib/featureFlag'
import { EmptyLibrary } from './EmptyLibrary'
import TopBarProgress from 'react-topbar-progress-indicator'
import { State } from '../../../lib/networking/fragments/articleFragment'
export type LayoutType = 'LIST_LAYOUT' | 'GRID_LAYOUT'
@ -258,7 +259,11 @@ export function HomeFeedContainer(props: HomeFeedContainerProps): JSX.Element {
const username = viewerData?.me?.profile.username
if (username) {
setActiveCardId(item.node.id)
router.push(`/${username}/${item.node.slug}`)
if (item.node.state === State.PROCESSING) {
router.push(`/${username}/links/${item.node.id}`)
} else {
router.push(`/${username}/${item.node.slug}`)
}
}
break
case 'showOriginal':

92
packages/web/lib/hooks/useReaderSettings.tsx Normal file
View File

@ -0,0 +1,92 @@
import { useCallback, useState } from "react"
import { userPersonalizationMutation } from "../networking/mutations/userPersonalizationMutation"
import { useGetUserPreferences, UserPreferences } from "../networking/queries/useGetUserPreferences"
import { usePersistedState } from "./usePersistedState"
export type ReaderSettings = {
preferencesData: UserPreferences | undefined
fontSize: number
lineHeight: number
marginWidth: number
setFontSize: (newFontSize: number) => void
setLineHeight: (newLineHeight: number) => void
setMarginWidth: (newMarginWidth: number) => void
showSetLabelsModal: boolean
showEditDisplaySettingsModal: boolean
setShowSetLabelsModal: (showSetLabelsModal: boolean) => void
setShowEditDisplaySettingsModal: (showEditDisplaySettingsModal: boolean) => void
actionHandler: (action: string, arg?: unknown) => void
}
export const useReaderSettings = (): ReaderSettings => {
const { preferencesData } = useGetUserPreferences()
const [fontSize, setFontSize] = useState(preferencesData?.fontSize ?? 20)
const [lineHeight, setLineHeight] = usePersistedState({ key: 'lineHeight', initialValue: 150 })
const [marginWidth, setMarginWidth] = usePersistedState({ key: 'marginWidth', initialValue: 200 })
const [showSetLabelsModal, setShowSetLabelsModal] = useState(false)
const [showEditDisplaySettingsModal, setShowEditDisplaySettingsModal] = useState(false)
const actionHandler = useCallback(async(action: string, arg?: unknown) => {
const updateFontSize = async(newFontSize: number) => {
setFontSize(newFontSize)
await userPersonalizationMutation({ fontSize: newFontSize })
}
switch (action) {
case 'incrementFontSize':
await updateFontSize(Math.min(fontSize + 2, 28))
break
case 'decrementFontSize':
await updateFontSize(Math.max(fontSize - 2, 10))
break
case 'setMarginWidth': {
const value = Number(arg)
if (value >= 200 && value <= 560) {
setMarginWidth(value)
}
break
}
case 'incrementMarginWidth':
setMarginWidth(Math.min(marginWidth + 45, 560))
break
case 'decrementMarginWidth':
setMarginWidth(Math.max(marginWidth - 45, 200))
break
case 'setLineHeight': {
const value = Number(arg)
if (value >= 100 && value <= 300) {
setLineHeight(arg as number)
}
break
}
case 'editDisplaySettings': {
setShowEditDisplaySettingsModal(true)
break
}
case 'setLabels': {
setShowSetLabelsModal(true)
break
}
case 'resetReaderSettings': {
updateFontSize(20)
setMarginWidth(290)
setLineHeight(150)
break
}
}
}, [fontSize, setFontSize, lineHeight,
setLineHeight, marginWidth, setMarginWidth])
return {
preferencesData,
fontSize, lineHeight, marginWidth,
setFontSize, setLineHeight, setMarginWidth,
showSetLabelsModal, showEditDisplaySettingsModal,
setShowSetLabelsModal, setShowEditDisplaySettingsModal,
actionHandler,
}
}

8
packages/web/lib/networking/fragments/articleFragment.ts
View File

@ -18,11 +18,18 @@ export const articleFragment = gql`
isArchived
description
linkId
state
}
`
export type ContentReader = 'WEB' | 'PDF'
export enum State {
SUCCEEDED = 'SUCCEEDED',
PROCESSING = 'PROCESSING',
FAILED = 'FAILED',
}
export type ArticleFragmentData = {
id: string
title: string
@ -40,4 +47,5 @@ export type ArticleFragmentData = {
isArchived: boolean
description: string
linkId?: string
state?: State
}

11
packages/web/lib/networking/queries/useGetArticleQuery.tsx
View File

@ -1,7 +1,11 @@
import { gql } from 'graphql-request'
import useSWRImmutable, { Cache } from 'swr'
import { makeGqlFetcher, RequestContext, ssrFetcher } from '../networkHelpers'
import { articleFragment, ContentReader } from '../fragments/articleFragment'
import {
articleFragment,
ContentReader,
State,
} from '../fragments/articleFragment'
import { Highlight, highlightFragment } from '../fragments/highlightFragment'
import { ScopedMutator } from 'swr/dist/types'
import { Label, labelFragment } from '../fragments/labelFragment'
@ -48,6 +52,7 @@ export type ArticleAttributes = {
highlights: Highlight[]
linkId: string
labels?: Label[]
state?: State
}
const query = gql`
@ -79,7 +84,6 @@ const query = gql`
${labelFragment}
`
export function useGetArticleQuery({
username,
slug,
@ -137,11 +141,10 @@ export const cacheArticle = (
})
}
export const removeItemFromCache = (
cache: Cache<unknown>,
mutate: ScopedMutator,
itemId: string,
itemId: string
) => {
try {
const mappedCache = cache as Map<string, unknown>

13
packages/web/lib/networking/queries/useGetArticleSavingStatus.tsx
View File

@ -30,6 +30,7 @@ type ArticleSavingRequestData = {
errorCode?: string
user?: UserData
article?: ArticleAttributes
slug?: string
}
type UserData = {
@ -71,6 +72,7 @@ export function useGetArticleSavingStatus({
...HighlightFields
}
}
slug
}
}
... on ArticleSavingRequestError {
@ -112,9 +114,16 @@ export function useGetArticleSavingStatus({
if (status === 'SUCCEEDED') {
const username =
articleSavingRequest?.articleSavingRequest?.user?.profile?.username
const slug = articleSavingRequest?.articleSavingRequest?.article?.slug
const slug = articleSavingRequest?.articleSavingRequest?.slug
const articleSlug =
articleSavingRequest?.articleSavingRequest?.article?.slug
if (username && slug) {
return { successRedirectPath: `/${username}/${slug}`, article: articleSavingRequest?.articleSavingRequest?.article }
return { successRedirectPath: `/${username}/${slug}` }
} else if (username && articleSlug) {
return {
successRedirectPath: `/${username}/${articleSlug}`,
article: articleSavingRequest?.articleSavingRequest?.article,
}
} else {
return { successRedirectPath: `/home` }
}

9
packages/web/lib/networking/queries/useGetLibraryItemsQuery.tsx
View File

@ -84,6 +84,7 @@ const libraryItemFragment = gql`
isArchived
description
linkId
state
labels {
...LabelFields
}
@ -110,6 +111,7 @@ export function useGetLibraryItemsQuery({
first: $first
after: $after
query: $query
includePending: true
) {
... on ArticlesSuccess {
edges {
@ -260,14 +262,15 @@ export function useGetLibraryItemsQuery({
if (res) {
showSuccessToast('Link unarchived', { position: 'bottom-right' })
} else {
showErrorToast('Error unarchiving link', { position: 'bottom-right' })
showErrorToast('Error unarchiving link', {
position: 'bottom-right',
})
}
})
break
case 'delete':
updateData(undefined)
deleteLinkMutation(item.node.id)
.then((res) => {
deleteLinkMutation(item.node.id).then((res) => {
if (res) {
showSuccessToast('Link removed', { position: 'bottom-right' })
} else {

1
packages/web/package.json
View File

@ -40,7 +40,6 @@
"react-colorful": "^5.5.1",
"react-dom": "^17.0.2",
"react-hot-toast": "^2.1.1",
"react-loading-skeleton": "^3.0.2",
"react-topbar-progress-indicator": "^4.1.1",
"react-twitter-widgets": "^1.10.0",
"swr": "^1.0.1",

257
packages/web/pages/[username]/[slug]/index.tsx
View File

@ -10,7 +10,6 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import { useKeyboardShortcuts } from '../../../lib/keyboardShortcuts/useKeyboardShortcuts'
import { articleKeyboardCommands, navigationCommands } from '../../../lib/keyboardShortcuts/navigationShortcuts'
import dynamic from 'next/dynamic'
import { useGetUserPreferences } from '../../../lib/networking/queries/useGetUserPreferences'
import { webBaseURL } from '../../../lib/appConfig'
import { Toaster } from 'react-hot-toast'
import { createHighlightMutation } from '../../../lib/networking/mutations/createHighlightMutation'
@ -28,7 +27,8 @@ import { useSWRConfig } from 'swr'
import { showErrorToast, showSuccessToast } from '../../../lib/toastHelpers'
import { SetLabelsModal } from '../../../components/templates/article/SetLabelsModal'
import { DisplaySettingsModal } from '../../../components/templates/article/DisplaySettingsModal'
import { usePersistedState } from '../../../lib/hooks/usePersistedState'
import { useReaderSettings } from '../../../lib/hooks/useReaderSettings'
import { SkeletonArticleContainer } from '../../../components/templates/article/SkeletonArticleContainer'
const PdfArticleContainerNoSSR = dynamic<PdfArticleContainerProps>(
@ -44,12 +44,7 @@ export default function Home(): JSX.Element {
const [showHighlightsModal, setShowHighlightsModal] = useState(false)
const { viewerData } = useGetViewerQuery()
const { preferencesData } = useGetUserPreferences()
const [fontSize, setFontSize] = useState(preferencesData?.fontSize ?? 20)
const [lineHeight, setLineHeight] = usePersistedState({ key: 'lineHeight', initialValue: 150 })
const [marginWidth, setMarginWidth] = usePersistedState({ key: 'marginWidth', initialValue: 200 })
const [showSetLabelsModal, setShowSetLabelsModal] = useState(false)
const [showEditDisplaySettingsModal, setShowEditDisplaySettingsModal] = useState(false)
const readerSettings = useReaderSettings()
const { articleData } = useGetArticleQuery({
username: router.query.username as string,
@ -68,11 +63,6 @@ export default function Home(): JSX.Element {
useKeyboardShortcuts(navigationCommands(router))
const actionHandler = useCallback(async(action: string, arg?: unknown) => {
const updateFontSize = async(newFontSize: number) => {
setFontSize(newFontSize)
await userPersonalizationMutation({ fontSize: newFontSize })
}
switch (action) {
case 'archive':
if (article) {
@ -105,50 +95,11 @@ export default function Home(): JSX.Element {
case 'showHighlights':
setShowHighlightsModal(true)
break
case 'incrementFontSize':
await updateFontSize(Math.min(fontSize + 2, 28))
default:
readerSettings.actionHandler(action, arg)
break
case 'decrementFontSize':
await updateFontSize(Math.max(fontSize - 2, 10))
break
case 'setMarginWidth': {
const value = Number(arg)
if (value >= 200 && value <= 560) {
setMarginWidth(value)
}
break
}
case 'incrementMarginWidth':
setMarginWidth(Math.min(marginWidth + 45, 560))
break
case 'decrementMarginWidth':
setMarginWidth(Math.max(marginWidth - 45, 200))
break
case 'setLineHeight': {
const value = Number(arg)
if (value >= 100 && value <= 300) {
setLineHeight(arg as number)
}
break
}
case 'editDisplaySettings': {
setShowEditDisplaySettingsModal(true)
break
}
case 'setLabels': {
setShowSetLabelsModal(true)
break
}
case 'resetReaderSettings': {
updateFontSize(20)
setMarginWidth(290)
setLineHeight(150)
break
}
}
}, [article, cache, mutate, router,
fontSize, setFontSize, lineHeight,
setLineHeight, marginWidth, setMarginWidth])
}, [article, cache, mutate, router, readerSettings])
useKeyboardShortcuts(
articleKeyboardCommands(router, async (action) => {
@ -156,87 +107,87 @@ export default function Home(): JSX.Element {
})
)
if (article && viewerData?.me) {
return (
<PrimaryLayout
pageTestId="home-page-tag"
scrollElementRef={scrollRef}
headerToolbarControl={
<ArticleActionsMenu
article={article}
layout='top'
lineHeight={lineHeight}
marginWidth={marginWidth}
showReaderDisplaySettings={article.contentReader != 'PDF'}
articleActionHandler={actionHandler}
/>
}
alwaysDisplayToolbar={article.contentReader == 'PDF'}
pageMetaDataProps={{
title: article.title,
path: router.pathname,
description: article.description,
return (
<PrimaryLayout
pageTestId="home-page-tag"
scrollElementRef={scrollRef}
headerToolbarControl={
<ArticleActionsMenu
article={article}
layout='top'
lineHeight={readerSettings.lineHeight}
marginWidth={readerSettings.marginWidth}
showReaderDisplaySettings={article?.contentReader != 'PDF'}
articleActionHandler={actionHandler}
/>
}
alwaysDisplayToolbar={article?.contentReader == 'PDF'}
pageMetaDataProps={{
title: article?.title ?? '',
path: router.pathname,
description: article?.description ?? '',
}}
>
<Script async src="/static/scripts/mathJaxConfiguration.js" />
<Script
async
id="MathJax-script"
src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"
/>
<Toaster />
<VStack distribution="between" alignment="center" css={{
position: 'fixed',
flexDirection: 'row-reverse',
top: '-120px',
left: 8,
height: '100%',
width: '35px',
'@lgDown': {
display: 'none',
},
}}
>
<Script async src="/static/scripts/mathJaxConfiguration.js" />
<Script
async
id="MathJax-script"
src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"
/>
<Toaster />
<VStack distribution="between" alignment="center" css={{
position: 'fixed',
flexDirection: 'row-reverse',
top: '-120px',
left: 8,
height: '100%',
width: '35px',
'@lgDown': {
display: 'none',
},
}}
>
{article.contentReader !== 'PDF' ? (
<ArticleActionsMenu
article={article}
layout='side'
lineHeight={lineHeight}
marginWidth={marginWidth}
showReaderDisplaySettings={true}
articleActionHandler={actionHandler}
/>
) : null}
</VStack>
{article.contentReader == 'PDF' ? (
<PdfArticleContainerNoSSR
article={article}
showHighlightsModal={showHighlightsModal}
setShowHighlightsModal={setShowHighlightsModal}
viewerUsername={viewerData.me?.profile?.username}
/>
) : (
<VStack
alignment="center"
distribution="center"
ref={scrollRef}
className="disable-webkit-callout"
css={{
'@smDown': {
background: theme.colors.grayBg.toString(),
}
}}
>
{article?.contentReader !== 'PDF' ? (
<ArticleActionsMenu
article={article}
layout='side'
lineHeight={readerSettings.lineHeight}
marginWidth={readerSettings.marginWidth}
showReaderDisplaySettings={true}
articleActionHandler={actionHandler}
/>
) : null}
</VStack>
{article && viewerData?.me && article.contentReader == 'PDF' ? (
<PdfArticleContainerNoSSR
article={article}
showHighlightsModal={showHighlightsModal}
setShowHighlightsModal={setShowHighlightsModal}
viewerUsername={viewerData.me?.profile?.username}
/>
) : (
<VStack
alignment="center"
distribution="center"
ref={scrollRef}
className="disable-webkit-callout"
css={{
'@smDown': {
background: theme.colors.grayBg.toString(),
}
}}
>
{article && viewerData?.me ? (
<ArticleContainer
article={article}
scrollElementRef={scrollRef}
isAppleAppEmbed={false}
highlightBarDisabled={false}
highlightsBaseURL={`${webBaseURL}/${viewerData.me?.profile?.username}/${slug}/highlights`}
fontSize={fontSize}
margin={marginWidth}
lineHeight={lineHeight}
fontSize={readerSettings.fontSize}
margin={readerSettings.marginWidth}
lineHeight={readerSettings.lineHeight}
labels={labels}
showHighlightsModal={showHighlightsModal}
setShowHighlightsModal={setShowHighlightsModal}
@ -248,30 +199,34 @@ export default function Home(): JSX.Element {
articleReadingProgressMutation,
}}
/>
</VStack>
) : (
<SkeletonArticleContainer
margin={readerSettings.marginWidth}
lineHeight={readerSettings.lineHeight}
fontSize={readerSettings.fontSize}
/>
)}
</VStack>
)}
{showSetLabelsModal && (
<SetLabelsModal
article={article}
linkId={article.id}
labels={article.labels}
articleActionHandler={actionHandler}
onOpenChange={() => setShowSetLabelsModal(false)}
/>
)}
{article && readerSettings.showSetLabelsModal && (
<SetLabelsModal
article={article}
linkId={article.id}
labels={article.labels}
articleActionHandler={actionHandler}
onOpenChange={() => readerSettings.setShowSetLabelsModal(false)}
/>
)}
{showEditDisplaySettingsModal && (
<DisplaySettingsModal
lineHeight={lineHeight}
marginWidth={marginWidth}
articleActionHandler={actionHandler}
onOpenChange={() => setShowEditDisplaySettingsModal(false)}
/>
)}
</PrimaryLayout>
)
}
return <LoadingView />
{readerSettings.showEditDisplaySettingsModal && (
<DisplaySettingsModal
lineHeight={readerSettings.lineHeight}
marginWidth={readerSettings.marginWidth}
articleActionHandler={actionHandler}
onOpenChange={() => readerSettings.setShowEditDisplaySettingsModal(false)}
/>
)}
</PrimaryLayout>
)
}

74
packages/web/pages/[username]/links/[id].tsx
View File

@ -6,19 +6,85 @@ import {
Loader,
ErrorComponent,
} from '../../../components/templates/SavingRequest'
import { ArticleActionsMenu } from '../../../components/templates/article/ArticleActionsMenu'
import { VStack } from '../../../components/elements/LayoutPrimitives'
import { theme } from '../../../components/tokens/stitches.config'
import { applyStoredTheme } from '../../../lib/themeUpdater'
import { useReaderSettings } from '../../../lib/hooks/useReaderSettings'
import { SkeletonArticleContainer } from '../../../components/templates/article/SkeletonArticleContainer'
import TopBarProgress from 'react-topbar-progress-indicator'
export default function ArticleSavingRequestPage(): JSX.Element {
const router = useRouter()
const readerSettings = useReaderSettings()
const [articleId, setArticleId] = useState<string | undefined>(undefined)
applyStoredTheme(false)
useEffect(() => {
if (!router.isReady) return
setArticleId(router.query.id as string)
}, [router.isReady, router.query.id])
return (
<PrimaryLayout pageTestId="saving-request-page-tag">
{articleId ? <PrimaryContent articleId={articleId} /> : <Loader />}
<PrimaryLayout
pageTestId="home-page-tag"
headerToolbarControl={
<ArticleActionsMenu
article={undefined}
layout='top'
lineHeight={readerSettings.lineHeight}
marginWidth={readerSettings.marginWidth}
showReaderDisplaySettings={true}
articleActionHandler={readerSettings.actionHandler}
/>
}
alwaysDisplayToolbar={false}
pageMetaDataProps={{
title: 'Saving link',
path: router.pathname,
}}
>
<TopBarProgress />
<VStack distribution="between" alignment="center" css={{
position: 'fixed',
flexDirection: 'row-reverse',
top: '-120px',
left: 8,
height: '100%',
width: '35px',
'@lgDown': {
display: 'none',
},
}}
>
<ArticleActionsMenu
article={undefined}
layout='side'
lineHeight={readerSettings.lineHeight}
marginWidth={readerSettings.marginWidth}
showReaderDisplaySettings={true}
articleActionHandler={readerSettings.actionHandler}
/>
</VStack>
<VStack
alignment="center"
distribution="center"
className="disable-webkit-callout"
css={{
'@smDown': {
background: theme.colors.grayBg.toString(),
}
}}
>
<SkeletonArticleContainer
margin={readerSettings.marginWidth}
fontSize={readerSettings.fontSize}
lineHeight={readerSettings.lineHeight}
>
{articleId ? <PrimaryContent articleId={articleId} /> : <Loader />}
</SkeletonArticleContainer>
</VStack>
</PrimaryLayout>
)
}
@ -59,5 +125,7 @@ function PrimaryContent(props: PrimaryContentProps): JSX.Element {
router.replace(successRedirectPath)
}
return <Loader />
return (
<Loader />
)
}

16
packages/web/pages/app/[username]/link-request/[id].tsx
View File

@ -7,8 +7,6 @@ import { useSWRConfig } from 'swr'
import { cacheArticle } from '../../../../lib/networking/queries/useGetArticleQuery'
import { PrimaryLayout } from '../../../../components/templates/PrimaryLayout'
import { applyStoredTheme } from '../../../../lib/themeUpdater'
import Skeleton, { SkeletonTheme } from 'react-loading-skeleton'
import 'react-loading-skeleton/dist/skeleton.css'
import { StyledText } from '../../../../components/elements/StyledText'
import { theme } from '../../../../components/tokens/stitches.config'
@ -47,19 +45,7 @@ export default function LinkRequestPage(): JSX.Element {
function Loader(): JSX.Element {
return (
<SkeletonTheme baseColor={theme.colors.grayText.toString()} highlightColor="#444">
<div style={{ fontSize: 20, lineHeight: 2 }}>
<StyledText style="boldHeadline"></StyledText>
<p></p>
<Skeleton count={3} />
<p></p>
<Skeleton count={2} />
<p></p>
<Skeleton count={5} />
<p></p>
<Skeleton count={5} />
</div>
</SkeletonTheme>
<Box />
)
}

74
packages/web/pages/article/sr/[id].tsx
View File

@ -6,19 +6,85 @@ import {
Loader,
ErrorComponent,
} from '../../../components/templates/SavingRequest'
import { ArticleActionsMenu } from '../../../components/templates/article/ArticleActionsMenu'
import { VStack } from '../../../components/elements/LayoutPrimitives'
import { theme } from '../../../components/tokens/stitches.config'
import { applyStoredTheme } from '../../../lib/themeUpdater'
import { useReaderSettings } from '../../../lib/hooks/useReaderSettings'
import { SkeletonArticleContainer } from '../../../components/templates/article/SkeletonArticleContainer'
import TopBarProgress from 'react-topbar-progress-indicator'
export default function ArticleSavingRequestPage(): JSX.Element {
const router = useRouter()
const readerSettings = useReaderSettings()
const [articleId, setArticleId] = useState<string | undefined>(undefined)
applyStoredTheme(false)
useEffect(() => {
if (!router.isReady) return
setArticleId(router.query.id as string)
}, [router.isReady, router.query.id])
return (
<PrimaryLayout pageTestId="saving-request-page-tag">
{articleId ? <PrimaryContent articleId={articleId} /> : <Loader />}
<PrimaryLayout
pageTestId="home-page-tag"
headerToolbarControl={
<ArticleActionsMenu
article={undefined}
layout='top'
lineHeight={readerSettings.lineHeight}
marginWidth={readerSettings.marginWidth}
showReaderDisplaySettings={true}
articleActionHandler={readerSettings.actionHandler}
/>
}
alwaysDisplayToolbar={false}
pageMetaDataProps={{
title: 'Saving link',
path: router.pathname,
}}
>
<TopBarProgress />
<VStack distribution="between" alignment="center" css={{
position: 'fixed',
flexDirection: 'row-reverse',
top: '-120px',
left: 8,
height: '100%',
width: '35px',
'@lgDown': {
display: 'none',
},
}}
>
<ArticleActionsMenu
article={undefined}
layout='side'
lineHeight={readerSettings.lineHeight}
marginWidth={readerSettings.marginWidth}
showReaderDisplaySettings={true}
articleActionHandler={readerSettings.actionHandler}
/>
</VStack>
<VStack
alignment="center"
distribution="center"
className="disable-webkit-callout"
css={{
'@smDown': {
background: theme.colors.grayBg.toString(),
}
}}
>
<SkeletonArticleContainer
margin={readerSettings.marginWidth}
fontSize={readerSettings.fontSize}
lineHeight={readerSettings.lineHeight}
>
{articleId ? <PrimaryContent articleId={articleId} /> : <Loader />}
</SkeletonArticleContainer>
</VStack>
</PrimaryLayout>
)
}
@ -59,5 +125,7 @@ function PrimaryContent(props: PrimaryContentProps): JSX.Element {
router.replace(successRedirectPath)
}
return <Loader />
return (
<Loader />
)
}

5
yarn.lock
View File

@ -20857,11 +20857,6 @@ react-is@^16.13.1, react-is@^16.7.0, react-is@^16.8.1:
resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4"
integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==
react-loading-skeleton@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/react-loading-skeleton/-/react-loading-skeleton-3.0.2.tgz#f1ba7344e03fb3363f2986e0f879d88249fc230f"
integrity sha512-rlALwuZEcjazUDeIy3+fEhm20Uk9Yd/zZGeITU034K2ts5/yEf7RuZaV2FyrPWypIII4LAsFEo9WDTPKp6G0fQ==
react-popper-tooltip@^3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/react-popper-tooltip/-/react-popper-tooltip-3.1.1.tgz#329569eb7b287008f04fcbddb6370452ad3f9eac"