Benature/omnivore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove redundant logs

Browse Source
This commit is contained in:
Hongbo Wu
2024-01-29 11:48:47 +08:00
parent 0181a2bfcd
commit d5d581fc54
18 changed files with 85 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
packages/api/src/events/reports/content_display_report_created.ts
View File

@ -23,16 +23,17 @@ export class ContentDisplayReportSubscriber
${report.user.id} for URL: ${report.originalUrl}
${report.reportComment}`
logger.info(message)
if (!env.dev.isLocal) {
// If we are in the local environment, just log a message, otherwise email the report
await sendEmail({
to: env.sender.feedback,
subject: 'New content display report',
text: message,
from: env.sender.message,
})
// If we are in the local environment, just log a message, otherwise email the report
if (env.dev.isLocal) {
logger.info(message)
return
}
await sendEmail({
to: env.sender.feedback,
subject: 'New content display report',
text: message,
from: env.sender.message,
})
}
}

10
packages/api/src/jobs/find_thumbnail.ts
View File

@ -5,7 +5,7 @@ import {
findLibraryItemById,
updateLibraryItem,
} from '../services/library_item'
import { createImageProxyUrl, createThumbnailUrl } from '../utils/imageproxy'
import { createThumbnailUrl } from '../utils/imageproxy'
import { logger } from '../utils/logger'
interface Data {
@ -22,7 +22,7 @@ interface ImageSize {
export const THUMBNAIL_JOB = 'find-thumbnail'
const fetchImage = async (url: string): Promise<AxiosResponse | null> => {
console.log('fetching image', url)
logger.info('fetching image', url)
try {
// get image file by url
return await axios.get(url, {
@ -70,7 +70,6 @@ export const fetchAllImageSizes = async (content: string) => {
// fetch all images by src and get their sizes
const images = dom.querySelectorAll('img[src]')
if (!images || images.length === 0) {
console.log('no images')
return []
}
@ -100,7 +99,6 @@ export const _findThumbnail = (imagesSizes: (ImageSize | null)[]) => {
// ignore small images
if (area < 5000) {
logger.info('ignore small', { src: imageSize.src })
continue
}
@ -109,13 +107,11 @@ export const _findThumbnail = (imagesSizes: (ImageSize | null)[]) => {
Math.max(imageSize.width, imageSize.height) /
Math.min(imageSize.width, imageSize.height)
if (ratio > 1.5) {
logger.info('penalizing long/wide', { src: imageSize.src })
area /= ratio * 2
}
// penalize images with "sprite" in their name
if (imageSize.src.toLowerCase().includes('sprite')) {
logger.info('penalizing sprite', { src: imageSize.src })
area /= 10
}
@ -139,7 +135,6 @@ export const findThumbnail = async (data: Data) => {
const thumbnail = item.thumbnail
if (thumbnail) {
logger.info('thumbnail already set')
const proxyUrl = createThumbnailUrl(thumbnail)
// pre-cache thumbnail first if exists
const image = await fetchImage(proxyUrl)
@ -154,7 +149,6 @@ export const findThumbnail = async (data: Data) => {
const imageSizes = await fetchAllImageSizes(item.readableContent)
// find thumbnail from all images if thumbnail not set
if (!item.thumbnail && imageSizes.length > 0) {
logger.info('finding thumbnail...')
const thumbnail = _findThumbnail(imageSizes)
if (!thumbnail) {
logger.info('no thumbnail found from content')

13
packages/api/src/jobs/rss/refreshAllFeeds.ts
View File

@ -5,6 +5,7 @@ import { getBackendQueue } from '../../queue-processor'
import { validateUrl } from '../../services/create_page_save_request'
import { RssSubscriptionGroup } from '../../utils/createTask'
import { stringToHash } from '../../utils/helpers'
import { logger } from '../../utils/logger'
export type RSSRefreshContext = {
type: 'all' | 'user-added'
@ -44,7 +45,7 @@ export const refreshAllFeeds = async (db: DataSource): Promise<boolean> => {
['RSS', 'ACTIVE', 'following', 'ACTIVE']
)) as RssSubscriptionGroup[]
console.log(`rss: checking ${subscriptionGroups.length}`, {
logger.info(`rss: checking ${subscriptionGroups.length}`, {
refreshContext,
})
@ -53,11 +54,11 @@ export const refreshAllFeeds = async (db: DataSource): Promise<boolean> => {
await updateSubscriptionGroup(group, refreshContext)
} catch (err) {
// we don't want to fail the whole job if one subscription group fails
console.error('error updating subscription group')
logger.error('error updating subscription group')
}
}
const finishTime = new Date()
console.log(
logger.info(
`rss: finished queuing subscription groups at ${finishTime.toISOString()}`,
{
refreshContext,
@ -74,18 +75,18 @@ const updateSubscriptionGroup = async (
let feedURL = group.url
const userList = JSON.stringify(group.userIds.sort())
if (!feedURL) {
console.error('no url for feed group', group)
logger.error('no url for feed group', group)
return
}
if (!userList) {
console.error('no userlist for feed group', group)
logger.error('no userlist for feed group', group)
return
}
try {
feedURL = validateUrl(feedURL).toString()
} catch (err) {
console.log('not refreshing invalid feed url: ', { feedURL })
logger.error('not refreshing invalid feed url: %s', feedURL)
}
const jobid = `refresh-feed_${stringToHash(feedURL)}_${stringToHash(
userList

60
packages/api/src/jobs/rss/refreshFeed.ts
View File

@ -97,11 +97,11 @@ const isFeedBlocked = async (feedUrl: string) => {
// if the feed has failed to fetch more than certain times, block it
const maxFailures = parseInt(process.env.MAX_FEED_FETCH_FAILURES ?? '10')
if (result && parseInt(result) > maxFailures) {
console.log('feed is blocked: ', feedUrl)
logger.info('feed is blocked: %s', feedUrl)
return true
}
} catch (error) {
console.error('Failed to check feed block status', feedUrl, error)
logger.error('Failed to check feed block status', { feedUrl, error })
}
return false
@ -117,7 +117,7 @@ const incrementFeedFailure = async (feedUrl: string) => {
return result
} catch (error) {
console.error('Failed to block feed', feedUrl, error)
logger.error('Failed to block feed', { feedUrl, error })
return null
}
}
@ -165,7 +165,7 @@ export const fetchAndChecksum = async (url: string) => {
return { url, content: dataStr, checksum: hash.digest('hex') }
} catch (error) {
console.log(`Failed to fetch or hash content from ${url}.`, error)
logger.info(`Failed to fetch or hash content from ${url}.`, error)
return null
}
}
@ -209,7 +209,7 @@ const parseFeed = async (url: string, content: string) => {
// otherwise the error will be caught by the outer try catch
return await parser.parseString(content)
} catch (error) {
console.log(error)
logger.info(error)
return null
}
}
@ -220,7 +220,7 @@ const isItemRecentlySaved = async (userId: string, url: string) => {
const result = await redisDataSource.redisClient?.get(key)
return !!result
} catch (err) {
console.error('error checking if item is old', err)
logger.error('error checking if item is old', err)
}
// If we failed to check, assume the item is good
return false
@ -256,7 +256,7 @@ const createTask = async (
) => {
const isRecentlySaved = await isItemRecentlySaved(userId, item.link)
if (isRecentlySaved) {
console.log('Item recently saved', item.link)
logger.info('Item recently saved %s', item.link)
return true
}
@ -264,7 +264,7 @@ const createTask = async (
return createItemWithPreviewContent(userId, feedUrl, item)
}
console.log(`adding fetch content task ${userId} ${item.link.trim()}`)
logger.info(`adding fetch content task ${userId} ${item.link.trim()}`)
return addFetchContentTask(fetchContentTasks, userId, folder, item)
}
@ -292,7 +292,7 @@ const fetchContentAndCreateItem = async (
})
return !!task
} catch (error) {
console.error('Error while creating task', error)
logger.error('Error while creating task', error)
return false
}
}
@ -334,7 +334,7 @@ const createItemWithPreviewContent = async (
})
return !!task
} catch (error) {
console.error('Error while creating task', error)
logger.error('Error while creating task', error)
return false
}
}
@ -454,7 +454,11 @@ const processSubscription = async (
let lastValidItem: RssFeedItem | null = null
if (fetchResult.checksum === lastFetchedChecksum) {
console.log('feed has not been updated', feedUrl, lastFetchedChecksum)
logger.info(
'feed has not been updated %s, %s',
feedUrl,
lastFetchedChecksum
)
return
}
const updatedLastFetchedChecksum = fetchResult.checksum
@ -464,12 +468,12 @@ const processSubscription = async (
failedAt: Date | undefined
const feedLastBuildDate = feed.lastBuildDate
console.log('Feed last build date', feedLastBuildDate)
logger.info('Feed last build date %s', feedLastBuildDate)
if (
feedLastBuildDate &&
new Date(feedLastBuildDate) <= new Date(mostRecentItemDate)
) {
console.log('Skipping old feed', feedLastBuildDate)
logger.info('Skipping old feed %s', feedLastBuildDate)
return
}
@ -522,7 +526,7 @@ const processSubscription = async (
// skip old items
if (isOldItem(feedItem, mostRecentItemDate)) {
console.log('Skipping old feed item', feedItem.link)
logger.info('Skipping old feed item %s', feedItem.link)
continue
}
@ -545,7 +549,7 @@ const processSubscription = async (
itemCount = itemCount + 1
} catch (error) {
console.error('Error while saving RSS feed item', { error, item })
logger.error('Error while saving RSS feed item', { error, item })
failedAt = new Date()
}
}
@ -554,7 +558,7 @@ const processSubscription = async (
if (!lastItemFetchedAt && !failedAt) {
// the feed has been fetched before, no new valid items found
if (mostRecentItemDate || !lastValidItem) {
console.log('No new valid items found')
logger.info('No new valid items found')
return
}
@ -568,7 +572,9 @@ const processSubscription = async (
folder
)
if (!created) {
console.error('Failed to create task for feed item', lastValidItem.link)
logger.error('Failed to create task for feed item', {
url: lastValidItem.link,
})
failedAt = new Date()
}
@ -589,14 +595,14 @@ const processSubscription = async (
refreshedAt,
failedAt,
})
console.log('Updated subscription', updatedSubscription)
logger.info('Updated subscription', updatedSubscription)
}
export const refreshFeed = async (request: any) => {
if (isRefreshFeedRequest(request)) {
return _refreshFeed(request)
}
console.log('not a feed to refresh')
logger.info('not a feed to refresh')
return false
}
@ -613,36 +619,36 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => {
refreshContext,
} = request
console.log('Processing feed', feedUrl, { refreshContext: refreshContext })
logger.info('Processing feed', feedUrl, { refreshContext: refreshContext })
try {
const isBlocked = await isFeedBlocked(feedUrl)
if (isBlocked) {
console.log('feed is blocked: ', feedUrl)
logger.info('feed is blocked: %s', feedUrl)
throw new Error('feed is blocked')
}
const fetchResult = await fetchAndChecksum(feedUrl)
if (!fetchResult) {
console.error('Failed to fetch RSS feed', feedUrl)
logger.error('Failed to fetch RSS feed %s', feedUrl)
await incrementFeedFailure(feedUrl)
throw new Error('Failed to fetch RSS feed')
}
const feed = await parseFeed(feedUrl, fetchResult.content)
if (!feed) {
console.error('Failed to parse RSS feed', feedUrl)
logger.error('Failed to parse RSS feed %s', feedUrl)
await incrementFeedFailure(feedUrl)
throw new Error('Failed to parse RSS feed')
}
let allowFetchContent = true
if (isContentFetchBlocked(feedUrl)) {
console.log('fetching content blocked for feed: ', feedUrl)
logger.info('fetching content blocked for feed: %s', feedUrl)
allowFetchContent = false
}
console.log('Fetched feed', feed.title, new Date())
logger.info('Fetched feed %s at %s', feed.title, new Date())
const fetchContentTasks = new Map<string, FetchContentTask>() // url -> FetchContentTask
// process each subscription sequentially
@ -664,7 +670,7 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => {
feed
)
} catch (error) {
console.error('Error while processing subscription', {
logger.error('Error while processing subscription', {
error,
subscriptionId,
})
@ -682,7 +688,7 @@ export const _refreshFeed = async (request: RefreshFeedRequest) => {
return true
} catch (error) {
console.error('Error while saving RSS feeds', {
logger.error('Error while saving RSS feeds', {
feedUrl,
subscriptionIds,
error,

16
packages/api/src/jobs/save_page.ts
View File

@ -71,7 +71,7 @@ const uploadToSignedUrl = async (
timeout: REQUEST_TIMEOUT,
})
} catch (error) {
console.error('error uploading to signed url', error)
logger.error('error uploading to signed url', error)
return null
}
}
@ -129,7 +129,7 @@ const sendImportStatusUpdate = async (
}
)
} catch (e) {
console.error('error while sending import status update', e)
logger.error('error while sending import status update', e)
}
}
@ -149,7 +149,7 @@ const getCachedFetchResult = async (url: string) => {
throw new Error('fetch result is not valid')
}
console.log('fetch result is cached', url)
logger.info('fetch result is cached', url)
return fetchResult
}
@ -172,7 +172,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
state = data.state
try {
console.log(`savePageJob: ${userId} ${url}`)
logger.info(`savePageJob: ${userId} ${url}`)
// get the fetch result from cache
const fetchedResult = await getCachedFetchResult(url)
@ -218,7 +218,7 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
}
if (!content) {
console.log('content is not fetched', url)
logger.info('content is not fetched', url)
// set the state to failed if we don't have content
content = 'Failed to fetch content'
state = ArticleSavingRequestStatus.Failed
@ -250,16 +250,16 @@ export const savePageJob = async (data: Data, attemptsMade: number) => {
isSaved = true
} catch (e) {
if (e instanceof Error) {
console.error('error while saving page', e.message)
logger.error('error while saving page: %s', e.message)
} else {
console.error('error while saving page', 'unknown error')
logger.error('error while saving page: unknown error')
}
throw e
} finally {
const lastAttempt = attemptsMade === MAX_ATTEMPTS - 1
if (lastAttempt) {
console.log('last attempt reached', data.url)
logger.info('last attempt reached %s', data.url)
}
if (taskId && (isSaved || lastAttempt)) {

3
packages/api/src/jobs/update_pdf_content.ts
View File

@ -1,5 +1,4 @@
import {
UpdateContentMessage,
isUpdateContentMessage,
updateContentForFileItem,
} from '../services/update_pdf_content'
@ -9,6 +8,6 @@ export const updatePDFContentJob = async (data: unknown): Promise<boolean> => {
if (isUpdateContentMessage(data)) {
return await updateContentForFileItem(data)
}
logger.log('update_pdf_content data is not a update message', { data })
logger.info('update_pdf_content data is not a update message', { data })
return false
}

2
packages/api/src/redis_data_source.ts
View File

@ -45,7 +45,7 @@ export class RedisDataSource {
await this.workerRedisClient?.quit()
await this.redisClient?.quit()
} catch (err) {
console.error('error while shutting down redis', err)
logger.error('error while shutting down redis', err)
}
}
}

1
packages/api/src/routers/svc/content.ts
View File

@ -13,7 +13,6 @@ export function contentServiceRouter() {
const router = express.Router()
router.post('/search', async (req, res) => {
logger.info('search req', req)
const { message: msgStr, expired } = readPushSubscription(req)
logger.info('read pubsub message', { msgStr, expired })

6
packages/api/src/routers/svc/following.ts
View File

@ -53,15 +53,13 @@ export function followingServiceRouter() {
const router = express.Router()
router.post('/save', async (req, res) => {
logger.info('save following item request', req.body)
if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) {
console.log('query does not include valid token')
logger.info('query does not include valid token')
return res.sendStatus(403)
}
if (!isSaveFollowingItemRequest(req.body)) {
console.error('Invalid request body', req.body)
logger.error('Invalid request body', req.body)
return res.status(400).send('INVALID_REQUEST_BODY')
}

6
packages/api/src/routers/svc/links.ts
View File

@ -29,7 +29,7 @@ const getPruneMessage = (msgStr: string): PruneMessage => {
return obj
}
} catch (err) {
console.log('error deserializing event: ', { msgStr, err })
logger.error('error deserializing event: ', { msgStr, err })
}
// default to prune following folder items older than 30 days
@ -43,7 +43,6 @@ export function linkServiceRouter() {
const router = express.Router()
router.post('/create', async (req, res) => {
logger.info('create link req', req)
const { message: msgStr, expired } = readPushSubscription(req)
logger.info('read pubsub message', { msgStr, expired })
@ -71,7 +70,6 @@ export function linkServiceRouter() {
userId: msg.userId,
url: msg.url,
})
logger.info('create link request', request)
res.status(200).send(request)
} catch (err) {
@ -81,8 +79,6 @@ export function linkServiceRouter() {
})
router.post('/prune', async (req, res) => {
logger.info('prune expired items in folder')
const { message: msgStr, expired } = readPushSubscription(req)
if (!msgStr) {

4
packages/api/src/routers/svc/rss_feed.ts
View File

@ -23,11 +23,11 @@ export function rssFeedRouter() {
if (redisDataSource.workerRedisClient) {
await queueRSSRefreshAllFeedsJob()
} else {
console.log('unable to fetchAll feeds, redis is not configured')
logger.info('unable to fetchAll feeds, redis is not configured')
return res.status(500).send('Expired')
}
} catch (error) {
logger.info('error fetching rss feeds', error)
logger.error('error fetching rss feeds', error)
return res.status(500).send('Internal Server Error')
}

2
packages/api/src/routers/svc/user.ts
View File

@ -23,7 +23,7 @@ const getCleanupMessage = (msgStr: string): CleanupMessage => {
return obj
}
} catch (err) {
console.log('error deserializing event: ', { msgStr, err })
logger.error('error deserializing event: ', { msgStr, err })
}
return {

21
packages/api/src/services/reports.ts
View File

@ -33,18 +33,19 @@ export const saveContentDisplayReport = async (
${report.user.id} for URL: ${report.originalUrl}
${report.reportComment}`
logger.info(message)
if (!env.dev.isLocal) {
// If we are in the local environment, just log a message, otherwise email the report
await sendEmail({
to: env.sender.feedback,
subject: 'New content display report',
text: message,
from: env.sender.message,
})
// If we are in the local environment, just log a message, otherwise email the report
if (env.dev.isLocal) {
logger.info(message)
return !!report
}
await sendEmail({
to: env.sender.feedback,
subject: 'New content display report',
text: message,
from: env.sender.message,
})
return !!report
}

1
packages/api/src/services/save_email.ts
View File

@ -75,7 +75,6 @@ export const saveEmail = async (
existingLibraryItem.id,
input.userId
)
logger.info('updated page from email', updatedLibraryItem)
return updatedLibraryItem
}

3
packages/api/src/services/save_page.ts
View File

@ -58,7 +58,6 @@ const shouldParseInBackend = (input: SavePageInput): boolean => {
return (
ALREADY_PARSED_SOURCES.indexOf(input.source) === -1 &&
FORCE_PUPPETEER_URLS.some((regex) => {
console.log('REGEX: ', regex)
return regex.test(input.url)
})
)
@ -241,7 +240,7 @@ export const parsedContentToLibraryItem = ({
rssFeedUrl?: string | null
folder?: string | null
}): DeepPartial<LibraryItem> & { originalUrl: string } => {
console.log('save_page: state', { url, state, itemId })
logger.info('save_page: state', { url, state, itemId })
return {
id: itemId || undefined,
slug,

3
packages/api/src/utils/createTask.ts
View File

@ -67,7 +67,7 @@ const createHttpTaskWithToken = async ({
> => {
// If there is no Google Cloud Project Id exposed, it means that we are in local environment
if (env.dev.isLocal || !project) {
console.error(
logger.error(
'error: attempting to create a cloud task but not running in google cloud.'
)
return null
@ -172,7 +172,6 @@ export const createAppEngineTask = async ({
}
logger.info('Sending task:')
logger.info(task)
// Send create task request.
const request = { parent: parent, task: task }
const [response] = await client.createTask(request)

1
packages/api/src/utils/helpers.ts
View File

@ -25,7 +25,6 @@ import {
SearchItem,
} from '../generated/graphql'
import { createPubSubClient } from '../pubsub'
import { redisDataSource } from '../redis_data_source'
import { validateUrl } from '../services/create_page_save_request'
import { updateLibraryItem } from '../services/library_item'
import { Merge } from '../util'

7
packages/readabilityjs/Readability.js
View File

@ -3036,16 +3036,16 @@ Readability.prototype = {
// detect language from the html content
const languages = (await cld.detect(content, { isHTML: true })).languages;
console.log('Detected languages: ', languages);
this.log('Detected languages: ', languages);
if (languages.length > 0) {
code = languages[0].code;
}
console.log('Getting language name from code: ', code);
this.log('Getting language name from code: ', code);
let lang = new Intl.DisplayNames(['en'], {type: 'language'});
return lang.of(code);
} catch (error) {
console.error('Failed to get language', error);
this.log('Failed to get language', error);
return 'English';
}
},
@ -3082,7 +3082,6 @@ Readability.prototype = {
this._removeScripts(this._doc);
this._prepDocument();
console.log(this._doc.body.innerHTML);
var metadata = this._getArticleMetadata(jsonLd);
this._articleTitle = metadata.title;