From b8f18fba813ba12267d621f526f506f6d631b19f Mon Sep 17 00:00:00 2001 From: xangelo Date: Thu, 12 Jun 2025 10:46:13 -0400 Subject: [PATCH] feat: feed can appear in reader and/or river view Feeds can now appear in reader, river, or both views. This is helpful for some feeds that update hundreds of times a day that should be "river only" or feeds that update monthly and are better in "reader" view --- knexfile.ts | 7 +- .../20250612142143_add_feed_view_flags.ts | 18 +++ package.json | 5 +- src/public/reader.html | 73 ++++++------ src/server.ts | 111 ++++++++++++------ src/types.ts | 2 + 6 files changed, 144 insertions(+), 72 deletions(-) create mode 100644 migrations/20250612142143_add_feed_view_flags.ts diff --git a/knexfile.ts b/knexfile.ts index bae408b..673cc38 100644 --- a/knexfile.ts +++ b/knexfile.ts @@ -6,6 +6,11 @@ module.exports = { client: 'better-sqlite3', connection: { filename: join(__dirname, 'data.db') - } + }, + migrations: { + directory: join(__dirname, 'migrations'), + extension: 'ts' + }, + useNullAsDefault: true }, }; diff --git a/migrations/20250612142143_add_feed_view_flags.ts b/migrations/20250612142143_add_feed_view_flags.ts new file mode 100644 index 0000000..7bfae16 --- /dev/null +++ b/migrations/20250612142143_add_feed_view_flags.ts @@ -0,0 +1,18 @@ +import type { Knex } from "knex"; + + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable('feeds', (table) => { + table.boolean('show_in_river').defaultTo(true).notNullable(); + table.boolean('show_in_reader').defaultTo(true).notNullable(); + }); +} + + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('feeds', (table) => { + table.dropColumn('show_in_river'); + table.dropColumn('show_in_reader'); + }); +} + diff --git a/package.json b/package.json index af4dbfe..5292afd 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,10 @@ "version": "0.0.1", "scripts": { "dev": "yarn nodemon src/server.ts", - "build": "yarn tsc && cp -R src/public dist/src" + "build": "yarn tsc && cp -R src/public dist/src", + "migrate": "knex migrate:latest --knexfile knexfile.ts", + "migrate:rollback": "knex migrate:rollback --knexfile knexfile.ts", + "migrate:make": "knex migrate:make --knexfile knexfile.ts" }, "dependencies": { "better-sqlite3": "^11.10.0", diff --git a/src/public/reader.html b/src/public/reader.html index c4798eb..e02043d 100644 --- a/src/public/reader.html +++ b/src/public/reader.html @@ -1,37 +1,39 @@ - - River - - - - - - - - -

River of News

- An auto-updating list of news -

- -

- - + + + River + + + + + + + + + +

River of News

+ An auto-updating list of news +

+ +

+ + - + + + \ No newline at end of file diff --git a/src/server.ts b/src/server.ts index ec9b4d5..50a8844 100644 --- a/src/server.ts +++ b/src/server.ts @@ -12,7 +12,12 @@ const db = knex({ debug: process.env.DEBUG === 'true', connection: { filename: join('./data.db') - } + }, + migrations: { + directory: join(__dirname, '../migrations'), + extension: 'ts' + }, + useNullAsDefault: true }); const parser = new Parser(); const app = express(); @@ -20,25 +25,25 @@ const app = express(); const FEED_REFRESH_RATE = parseInt(process.env.FEED_REFRESH_RATE || '0'); function timestamp(obj: any): number { - if(obj.getTime) { - return Math.floor(obj.getTime()/1000) + if (obj.getTime) { + return Math.floor(obj.getTime() / 1000) } - else if(obj.toString() === obj) { + else if (obj.toString() === obj) { return Math.floor(new Date(obj).getTime() / 1000) } - else if(!isNaN(obj)) { - if(obj.toString().length === 10) { + else if (!isNaN(obj)) { + if (obj.toString().length === 10) { return obj; } else { - return Math.floor(obj/1000); + return Math.floor(obj / 1000); } } } function contentExtractor(feed: FeedSchema, item): string { let str = ''; - switch(feed.title) { + switch (feed.title) { case 'TechCrunch': str = item.contentSnippet; break; @@ -47,7 +52,7 @@ function contentExtractor(feed: FeedSchema, item): string { break; case 'The Register': str = item.summary; - break; + break; } str = str.replace(/<[^>]+>/g, ''); @@ -67,14 +72,14 @@ async function queryFeeds() { console.log(`Querying ${feedsToQuery.length} feeds`); - for(let feed of feedsToQuery) { + for (let feed of feedsToQuery) { console.log(`Querying ${feed.title}(${feed.url})`); try { const data = await parser.parseURL(feed.url); const items: FeedEntrySchema[] = data.items.map(item => { const id = createHash('sha256'); - id.update(item.guid || item.id || Date.now()+Math.random()); + id.update(item.guid || item.id || Date.now() + Math.random()); return { id: id.digest('hex'), title: item.title, @@ -92,7 +97,7 @@ async function queryFeeds() { } }); - for(let item of items) { + for (let item of items) { // inserting one at a time so that we can ignore duplicates for now // eventually we'll need to do some kind of merge await db('feed_entry').insert(item).onConflict().ignore(); @@ -106,7 +111,7 @@ async function queryFeeds() { } - catch(e) { + catch (e) { console.log(e); console.log('Continuing..'); } @@ -118,6 +123,24 @@ async function queryFeeds() { setTimeout(queryFeeds, 1000 * 60); } +// Migration runner function +async function runMigrations() { + try { + console.log('Running database migrations...'); + const [batchNo, log] = await db.migrate.latest(); + + if (log.length === 0) { + console.log('Database is already up to date'); + } else { + console.log(`Batch ${batchNo} run: ${log.length} migrations`); + log.forEach((migration) => console.log(`- ${migration}`)); + } + } catch (error) { + console.error('Migration failed:', error); + process.exit(1); + } +} + app.use(express.static(join(__dirname, 'public'))); app.use(express.json()); @@ -134,9 +157,9 @@ app.get('/river', async (req, res) => { const freshReadInHours = parseInt(process.env.FRESH_READ_IN_HOURS) || 6; let createdAt = Math.floor(timestamp(new Date()) - ((60 * 60) * freshReadInHours)); - if(!displayedItems) { + if (!displayedItems) { const temp = parseInt(header); - if(!isNaN(temp)) { + if (!isNaN(temp)) { firstLoad = false; createdAt = temp; } @@ -156,7 +179,7 @@ app.get('/river', async (req, res) => { .limit(100) .orderBy('feed_entry.pub_date', 'desc'); - if(entries[0] && entries[0].created_at) { + if (entries[0] && entries[0].created_at) { res.append('last-check', entries[0].created_at.toString()); } else { @@ -169,7 +192,7 @@ app.get('/river', async (req, res) => { let groupIndex = -1; entries.forEach(entry => { - if(entry.feed_id !== prevId) { + if (entry.feed_id !== prevId) { prevId = entry.feed_id; groupIndex++; groups.push([]); @@ -186,7 +209,7 @@ ${group.map(renderFeedItem).join("\n")} }); app.post('/feeds', async (req, res) => { - if(req.body.key !== process.env.ADMIN_KEY) { + if (req.body.key !== process.env.ADMIN_KEY) { res.sendStatus(400).end(); return; } @@ -205,17 +228,22 @@ app.post('/feeds', async (req, res) => { }); app.get('/feeds', async (req, res) => { - const feedList: FeedSchemaWithUnread[] = await db.raw(` -select - f.*, - sum(fe.is_read) as read, - count(fe.feed_id) as total -from feeds f -join feed_entry fe on fe.feed_id = f.id -group by fe.feed_id -`); + const filter = req.query.filter; + let query = ` + select + f.*, + sum(fe.is_read) as read, + count(fe.feed_id) as total + from feeds f + join feed_entry fe on fe.feed_id = f.id + group by fe.feed_id + `; + if (filter === 'reader') { + query += ` having f.show_in_reader = 1`; + } + const feedList: FeedSchemaWithUnread[] = await db.raw(query); - if(req.accepts('html')) { + if (req.accepts('html')) { res.send(renderReaderAppFeedList(feedList)); return; } @@ -241,7 +269,7 @@ where f.id = ? group by fe.feed_id `, [req.params.feed_id]); - if(req.accepts('html')) { + if (req.accepts('html')) { res.send(renderReaderAppFeedEntries(page, feedData.pop(), feedEntries)) return; } @@ -251,7 +279,7 @@ group by fe.feed_id app.post('/feed_entry/:feed_entry_id', async (req, res) => { const authSecret = req.header('x-secret'); - if(authSecret === process.env.ADMIN_KEY) { + if (authSecret === process.env.ADMIN_KEY) { const item: FeedWithEntrySchema[] = await db('feed_entry').update({ is_read: true }).where({ @@ -266,7 +294,7 @@ app.post('/feed_entry/:feed_entry_id', async (req, res) => { }); app.delete('/feeds/:feed_id', async (req, res) => { - if(req.query.key !== process.env.ADMIN_KEY) { + if (req.query.key !== process.env.ADMIN_KEY) { res.sendStatus(400).end(); return; } @@ -277,9 +305,22 @@ app.delete('/feeds/:feed_id', async (req, res) => { res.json(await db.select('*').from('feeds').orderBy('created_at')); }); -app.listen(process.env.API_PORT, () => { - console.log(`Listening on port ${process.env.API_PORT}`); - console.log(`Feed refresh rate: ${FEED_REFRESH_RATE}m`); - queryFeeds(); +// Start the application +async function startApp() { + // Run migrations first + await runMigrations(); + + // Then start the server + app.listen(process.env.API_PORT, () => { + console.log(`Listening on port ${process.env.API_PORT}`); + console.log(`Feed refresh rate: ${FEED_REFRESH_RATE}m`); + queryFeeds(); + }); +} + +// Start the application +startApp().catch(error => { + console.error('Failed to start application:', error); + process.exit(1); }); diff --git a/src/types.ts b/src/types.ts index abb7549..eeb42f2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -8,6 +8,8 @@ export type FeedSchema = { url: string; title: string; favicon: string; + show_in_river: boolean; + show_in_reader: boolean; } & KnexTimestamps; export type FeedSchemaWithUnread = FeedSchema & { -- 2.25.1