Feed Reader

feeds.js 165 lines

// Feed subscriptions live in SQL, with stable IDs shared by their items. The reader can fetch
// only sites the user allowed: subscribing asks Lattice for the feed's site
// (lattice.net.requestSite), and refreshing never asks.
import { parseFeed } from "./rss.js";

// Newest items imported from a feed per check.
const MAX_ITEMS = 100;
// Opening the reader checks feeds not checked for this long.
export const STALE_MS = 30 * 60 * 1000;
const MAX_REDIRECTS = 3;

let feeds = [];
const listeners = new Set();
let queue = Promise.resolve();

export const feedList = {
  subscribe(listener) {
    listeners.add(listener);
    return () => listeners.delete(listener);
  },
  getSnapshot: () => feeds,
};

export async function loadFeeds(store) {
  await store.ready;
  await readFeeds();
}

async function readFeeds() {
  feeds = await window.lattice.db.query("SELECT * FROM feeds WHERE subscribed=1 ORDER BY id");
  listeners.forEach((listener) => listener());
}

// Serialize subscription edits and publish a fresh SQL snapshot after each write.
function change(operation) {
  const run = queue.then(async () => {
    const result = await operation();
    await readFeeds();
    return result;
  });
  queue = run.catch(() => {});
  return run;
}

export function feedURL(input) {
  let value = input.trim();
  if (!/^[a-z][a-z0-9+.-]*:/i.test(value)) value = `https://${value}`;
  let url;
  try {
    url = new URL(value);
  } catch {
    throw new Error(
      "Enter a feed address such as https://example.com/feed.xml.",
    );
  }
  if (url.protocol === "http:") url.protocol = "https:";
  if (url.protocol !== "https:")
    throw new Error("Enter an https feed address.");
  return url.href;
}

export function siteName(url) {
  try {
    return new URL(url).host;
  } catch {
    return url;
  }
}

async function allow(url) {
  const { granted, site } = await window.lattice.net.requestSite(url);
  if (!granted) throw new Error(`You did not allow ${site}.`);
}

// GET a feed, following redirects to https addresses. With `ask`, each new
// site is put to the user first; without it, a site that is not allowed
// fails the check.
async function download(url, ask) {
  let current = url;
  for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
    if (ask) await allow(current);
    let response;
    try {
      response = await window.lattice.net.fetch(current);
    } catch (error) {
      const message = String(error.message || error);
      throw new Error(
        message.includes("not in the allowlist")
          ? `${siteName(current)} is not allowed. Subscribe to the feed again to allow it.`
          : message,
      );
    }
    const location = response.headers?.location;
    if (response.status >= 300 && response.status < 400 && location) {
      current = feedURL(new URL(location, current).href);
      continue;
    }
    if (response.status !== 200)
      throw new Error(
        `${siteName(current)} answered with status ${response.status}.`,
      );
    const body = response.body;
    return {
      url: current,
      xml: typeof body === "string" ? body : JSON.stringify(body),
    };
  }
  throw new Error("The feed address redirects too many times.");
}

// RSS <source> identifies original publication, independently of the subscription.
export function feedRows(feedId, feed) {
  return feed.items.slice(0, MAX_ITEMS).map((item) => ({ ...item, feed_id: feedId }));
}

async function saveSubscription(url, title) {
  return change(async () => {
    const [entry] = await window.lattice.db.query(
      `INSERT INTO feeds (url,title,subscribed) VALUES (?,?,1)
       ON CONFLICT(url) DO UPDATE SET title=excluded.title,subscribed=1 RETURNING *`,
      [url, title || siteName(url)]);
    return entry;
  });
}

export async function subscribe(store, input) {
  await store.ready;
  const { url, xml } = await download(feedURL(input), true);
  const feed = parseFeed(xml);
  const entry = await saveSubscription(url, feed.title);
  const counts = await store.importItems(feedRows(entry.id, feed));
  await change(() => window.lattice.db.exec(
    "UPDATE feeds SET checked_at=?,error=NULL WHERE id=?", [new Date().toISOString(), entry.id]));
  return { title: entry.title, added: counts.added };
}

// Keep the subscription row so saved items still identify their feed.
export function unsubscribe(url) {
  return change(() => window.lattice.db.exec("UPDATE feeds SET subscribed=0 WHERE url=?", [url]));
}

export async function refreshFeeds(store, { olderThan = 0 } = {}) {
  await loadFeeds(store);
  const due = feeds.filter(
    (feed) => Date.now() - (Date.parse(feed.checked_at) || 0) >= olderThan,
  );
  let added = 0, failed = 0;
  for (const entry of due) {
    try {
      const feed = parseFeed((await download(entry.url, false)).xml);
      // Update the visible title before importing so emitted events use it too.
      await change(() => window.lattice.db.exec(
        "UPDATE feeds SET title=? WHERE id=?", [feed.title || entry.title, entry.id]));
      added += (await store.importItems(feedRows(entry.id, feed))).added;
      await change(() => window.lattice.db.exec(
        "UPDATE feeds SET checked_at=?,error=NULL WHERE id=?", [new Date().toISOString(), entry.id]));
    } catch (error) {
      failed++;
      await change(() => window.lattice.db.exec(
        "UPDATE feeds SET checked_at=?,error=? WHERE id=?",
        [new Date().toISOString(), String(error.message || error), entry.id]));
    }
  }
  return { checked: due.length, added, failed };
}