diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 5f3adfd82..d1846ad51 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -432,9 +432,12 @@ export const isProbablyNewsletter = (html: string): boolean => { return true } - const beehiivUrl = beehiivNewsletterHref(dom.window) - if (beehiivUrl) { - return true + // Check if this is a beehiiv.net newsletter + if (dom.document.querySelectorAll('img[src*="beehiiv.net"]').length > 0) { + const beehiivUrl = beehiivNewsletterHref(dom.window) + if (beehiivUrl) { + return true + } } return false @@ -446,7 +449,6 @@ const beehiivNewsletterHref = (dom: DOMWindow): string | undefined => { ) let res: string | undefined = undefined readOnline.forEach((e) => { - console.log('text content', e.textContent) if (e.textContent === 'Read Online') { res = e.getAttribute('href') || undefined } @@ -455,10 +457,18 @@ const beehiivNewsletterHref = (dom: DOMWindow): string | undefined => { } const findNewsletterHeaderHref = (dom: DOMWindow): string | undefined => { + // Substack header links const postLink = dom.document.querySelector('h1 a ') if (postLink) { return postLink.getAttribute('href') || undefined } + + // Check if this is a beehiiv.net newsletter + const beehiiv = beehiivNewsletterHref(dom.window) + if (beehiiv) { + return beehiiv + } + return undefined } @@ -468,6 +478,8 @@ export const findNewsletterUrl = async ( html: string ): Promise => { const dom = new JSDOM(html).window + + // Check if this is a substack newsletter const href = findNewsletterHeaderHref(dom.window) if (href) { // Try to make a HEAD request so we get the redirected URL, since these diff --git a/packages/api/test/utils/data/beehiiv-newsletter.html b/packages/api/test/utils/data/beehiiv-newsletter.html index 61b2202f7..369d42af0 100644 --- a/packages/api/test/utils/data/beehiiv-newsletter.html +++ b/packages/api/test/utils/data/beehiiv-newsletter.html @@ -1,863 +1,15 @@ -I talked to a guy that spe= -nt $30M on a Beeple
= -
= -
Mar= -ch 7, 2022  |  Read Online
<= -/td>
= - 3D""
= -
3D""
3D""
<= -/table>
GM, this is the Milk Road. The daily email that tells you what=E2= -=80=99s going on in crypto, in less than 5 minutes. 
<= -div class=3D'paragraphText' style=3D'text-align:left;font-size:16px;font-fa= -mily:Verdana;line-height:22px;color:#222222;'> Markets are turbulent. Stock= -s are down. Oil is up. Wheat & corn prices are going crazy.
= -
Bitcoin holding at $39K. On= -ly 5 of the top 100 cryptocurrencies (by market cap) were in the green this= - weekend.
= -What's In Today=E2=80=99s Milk Road:
  • =F0=9F=A4=91 We called a crypto OG worth $= -100m+
  • =F0=9F=93=8A An= - interesting chart
  • = -=F0=9F=A5=9B Five Nuggets you'll wanna read
  • =F0=9F=A7=91=E2=80=8D=E2=9A=96=EF=B8=8F Coinbase and= - FTX talk about enforcing sanctions
3D""

W= -e talked to a Crypto OG worth $100m+

Just had a great call with Ryan Zur= -rer.
= -
4 Baller Things About Ryan: 
  1. He recentl= -y bought a $30M Beeple NFT  
  2. Early at Polychain, one of the first-ever crypto focused funds= - 
  3. His net worth = -is $100M+ 
  4. Early= - investor in MakerDAO, Filecoin, Axie Infinity, dydx, etc.
<= -/tr>
Highlights from the ca= -ll: 
= - He bought Human One by Beeple (a 7-foot sculpture made of LED screens) for= - $29m in November because it's a living and evolving piece of art, it's fun= -ny, and he believes Metcalfe's Law applies to NFTs.
(If you wanna see how it evolves, t= -he sculpture just updated to support Ukraine).
He's an avid NFT collector and sees something most people don= -'t: these pieces of digital art can earn him yield.
Here's how it works:
He borrows against the NFT= - in U.S. dollars, then uses the capital as a lender in a riskier, higher-in= -terest DeFi transaction (I.E. the 20% UST yield we showed you on Friday). <= -/div>
Just last wee= -k we saw someone get a loan for 101 CryptoPunks, so w= -e expect to see this more and more.
Here=E2=80=99s some other things he mentioned h= -e=E2=80=99s bullish on right now: 
Ronin
Axie Infinity was the first big breakout cry= -pto game. It generated like $2B in less than a year, and thousands of peopl= -e in emerging markets literally get paid to play the game as a job (=E2=80= -=9CPlay to Earn=E2=80=9D).
It runs on the Ronin blockchain, which has it=E2=80=99s own = -token $RON
I= -t's usage is so impressive, it's right up there with Ethereum's. Ronin has = -410k daily users, while Ethereum has 480k.
But Ronin has only a $2B market cap, while E= -thereum is ~$300B.
As Ryan put it: =E2=80=9CEither this is all just one giant internet= - casino and fundamentals don=E2=80=99t matter=E2=80=A6or fundamentals do= - matter and Ronin is grossly undervalued."
Safe Haven Assets: 
He named a bunch, but= - one non-crypto asset that stood out was 1st edition Charizard cards that a= -re in mint condition.
Pokemon is the highest grossing franchise of all time globally, a= -nd has lasted for decades and the 1st edition cards have a capped supply. <= -/div>
He has accumu= -lated a large collection of premium Charizard cards in his portfolio.
DeFi Yield<= -/div>
He and his te= -am built a product to deliver consistent yield farming results (they call i= -t Medici) in the 20-50% APR range.
You can read about it here
His advice for us: lean into what the nerds are doing&nbs= -p;
Ryan = -got into some great early deals by hustling & spending time with builders b= -efore they even had a token.  
Joining discords, participating in DAOs, goi= -ng to hackathons, conferences, brainstorming with devs etc. 
This is one way to ge= -t massive (1000x) types of returns.
If you wanna keep up with Ryan, here's his= - twitter and his company's blog.
3D""

Chart Of The Day =F0=9F=93=8A 

<= -table border=3D'0' cellpadding=3D'0' cellspacing=3D'0' role=3D'presentation= -' style=3D'border-collapse:collapse;' width=3D'100%'>
Andre Cronje is a famous crypto engin= -eer. He helped build Fantom, Yearn Finance, and a bunch of other prominent = -DeFi projects.
That=E2=80=99s why it was very surprising that he suddenly changed his L= -inkedIn and his partner tweeted they were done with building in crypto.
= -
3D""
This caused the price of Fantom to dr= -op 20% fast
-=3D""
<= -tr>
While the price = -has started to recover, this brings up an interesting question.  In a = -world of decentralization, how much should the founders involvement matter = -once there's real users and usage?
3D""

Five N= -uggets =F0=9F=94=8D

Jake Paul, Soulja Boy, & Lil Yachty are getting sued for promoting Safemoon coin
We bought a piece of a = -lamborghini that was blown up : the artist SHL0MS blew up a $250K lambo= - and auctioned off the fragments for ~$2M total. The auction la= -sted almost 30 hours and had 1784 total bidders with a top bid of 69 ETH (~= -$175k). Milk Road got in on the action & won CAR // 0883. See the top 50 bi= -dders here.
DeFi legends are leaving crypto and handing over  = -~25 apps and services: Andre Cronje and Anton Nell abruptly announ= -ce their decision to stop their work in the space. Tokens associated with t= -he projects are all down big - Fantom, Yearn.Finance, Solidly to name a few= -. This is how = -they broke the news.
I=E2=80=99ll take a side of Bitcoin with that burger please:= - Burger chain Shake Shack has announced that for every purchase made w= -ith a Cash App debit card through March, it will refund customers 15% of th= -e equivalent total in Bitcoin.
Tai Lopez NFT chaos: Tai was the talk of NFT = -land with his recent drop where he was auctioned off activities to do with = -him. Some of our favorites include: game of HORSE with Tai (18.4ETH =3D ~$4= -6,500 ), 1 on 1 Watch A Movie with Tai (12.27 ETH =3D ~$31,000) and 1 on 1 = -Shadow Tai At His Office (42.93 ETH =3D ~$108,400). Curious what else was f= -or sale? check it out here.
Terra on a roll: The latest ri= -se in the price of LUNA, Terra's native token, has made Terra the second-la= -rgest home for all things DeFi, right behind Ethereum. Check out the charts= - here.
3D""

How FTX & Coinbase Enforce US Sanctions

Both FTX and Coinbase put out threads= - on how they're enforcing US sanctions in the past few days:
FTX's Brett Harrison on sanctions
= -
Coinbase's Brian Armstrong on sanctions
Both are good reads t= -o understand how centralized exchanges are following the laws and how it mi= -ght not be a bad idea to hold your own keys from now on!
3D""

Share Milk Road

You currently h= -ave 0 referrals, only 1 away from receivi= -ng An Inside Look At What The Crypto Whales Are Betting On= -.
<= -!--[if mso]>

Click to Share

= -Or copy and paste this link to others: = -https://www.milkroad.com/subscribe?ref=3DKG6cY5h23f

Shar= -e Milk Road

<= -td align=3D'left' style=3D'padding-top:10px;padding-bottom:10px;'>
You currently have 0 referrals, only 1 away from receiving An Inside = -Look At What The Crypto Whales Are Betting On.

Click to Share

Or copy and paste this link to others: https= -://www.milkroad.com/subscribe?ref=3DKG6cY5h23f
= -

3D""
Alright, that's all we got for the start of the week!
Shaan "Chocolate Mi= -lk" Puri and Ben "2% Milk" Levy
What'd you think of today's emai= -l?
3D""

= -

<= -img width=3D"24" height=3D"24" style=3D"width:24px;height:24px;border:0;" a= -lt=3D"Twitter Logo" src=3D"https://media.beehiiv.net/static_assets/twitter_= -light.png"/>
© Mi= -lk Road   Unsubscribe

228 Park Ave S, #29976, New York, New Yo= -rk 10003

<= -img alt=3D'beehiiv logo' src=3D'https://d2ix49g14qmj6o.cloudfront.net/outpu= -t-onlinepngtools.png' style=3D'width:100%;max-width:25px;margin:auto;text-a= -lign:center;border:0;' width=3D'25'> Publish on beehiiv
\ No newline at end of file +I talked to a guy that spent $30M on a Beeple + +
+
+ + + diff --git a/packages/api/test/utils/parser.test.ts b/packages/api/test/utils/parser.test.ts index 63a7fa0a4..22ee7a46c 100644 --- a/packages/api/test/utils/parser.test.ts +++ b/packages/api/test/utils/parser.test.ts @@ -35,9 +35,10 @@ describe('findNewsletterUrl', async () => { // Not sure if the redirects from substack expire, this test could eventually fail expect(url).to.startWith('https://newsletter.slowchinese.net/p/companies-that-eat-people-217') }) - it('gets the URL from the header if it is a beehiiv newsletter', () => { + it('gets the URL from the header if it is a beehiiv newsletter', async () => { const html = load('./test/utils/data/beehiiv-newsletter.html') - isProbablyNewsletter(html).should.be.true + const url = await findNewsletterUrl(html) + expect(url).to.startWith('https://www.milkroad.com/p/talked-guy-spent-30m-beeple') }) it('returns undefined if it is not a newsletter', async () => { const html = load('./test/utils/data/substack-forwarded-welcome-email.html')