--- a/mod_pubsub_feeds/mod_pubsub_feeds.lua	Mon Sep 18 08:22:07 2023 -0500
+++ b/mod_pubsub_feeds/mod_pubsub_feeds.lua	Mon Sep 18 08:24:19 2023 -0500
@@ -1,17 +1,4 @@
 -- Fetches Atom feeds and publishes to PubSub nodes
---
--- Config:
--- Component "pubsub.example.com" "pubsub"
--- modules_enabled = {
---   "pubsub_feeds";
--- }
--- feeds = { -- node -> url
---   prosody_blog = "http://blog.prosody.im/feed/atom.xml";
--- }
--- feed_pull_interval = 20 -- minutes
---
--- Reference
--- http://pubsubhubbub.googlecode.com/svn/trunk/pubsubhubbub-core-0.4.html
 
 local pubsub = module:depends"pubsub";
 
@@ -36,7 +23,7 @@
 	return nil, "unsupported-format";
 end
 
-local use_pubsubhubub = module:get_option_boolean("use_pubsubhubub", true);
+local use_pubsubhubub = module:get_option_boolean("use_pubsubhubub", false);
 if use_pubsubhubub then
 	module:depends"http";
 end
@@ -46,7 +33,8 @@
 local formencode = http.formencode;
 
 local feed_list = module:shared("feed_list");
-local refresh_interval = module:get_option_number("feed_pull_interval", 15) * 60;
+local legacy_refresh_interval = module:get_option_number("feed_pull_interval", 15);
+local refresh_interval = module:get_option_number("feed_pull_interval_seconds", legacy_refresh_interval*60);
 local lease_length = tostring(math.floor(module:get_option_number("feed_lease_length", 86400)));
 
 function module.load()
@@ -60,7 +48,12 @@
 		end
 		new_feed_list[node] = true;
 		if not feed_list[node] then
-			feed_list[node] = { url = url; node = node; last_update = 0 };
+			local ok, err = pubsub.service:create(node, true);
+			if ok or err == "conflict" then
+				feed_list[node] = { url = url; node = node; last_update = 0 };
+			else
+				module:log("error", "Could not create node %s: %s", node, err);
+			end
 		else
 			feed_list[node].url = url;
 		end
@@ -75,58 +68,68 @@
 	end
 end
 
-function update_entry(item)
+function update_entry(item, data)
 	local node = item.node;
-	module:log("debug", "parsing %d bytes of data in node %s", #item.data or 0, node)
-	local feed, err = parse_feed(item.data);
+	module:log("debug", "parsing %d bytes of data in node %s", #data or 0, node)
+	local feed, err = parse_feed(data);
 	if not feed then
 		module:log("error", "Could not parse feed %q: %s", item.url, err);
-		module:log("debug", "Feed data:\n%s\n.", item.data);
+		module:log("debug", "Feed data:\n%s\n.", data);
 		return;
 	end
 	local entries = {};
 	for entry in feed:childtags("entry") do
 		table.insert(entries, entry);
 	end
-	local ok, items = pubsub.service:get_items(node, true);
+	local ok, last_id = pubsub.service:get_last_item(node, true);
 	if not ok then
-		local ok, err = pubsub.service:create(node, true);
-		if not ok then
-			module:log("error", "Could not create node %s: %s", node, err);
-			return;
+		module:log("error", "PubSub node %q missing: %s", node, last_id);
+		return
+	end
+
+	local start_from = #entries;
+	for i, entry in ipairs(entries) do
+		local id = entry:get_child_text("id");
+		if not id then
+			local link = entry:get_child("link");
+			if link then
+				module:log("debug", "Feed %q item %s is missing an id, using <link> instead", item.url, entry:top_tag());
+				id = link and link.attr.href;
+			else
+				module:log("error", "Feed %q item %s is missing both id and link, this feed is unusable", item.url, entry:top_tag());
+				return;
+			end
+			entry:text_tag("id", id);
 		end
-		items = {};
+
+		if last_id == id then
+			-- This should be the first item that we already have.
+			start_from = i-1;
+			break
+		end
 	end
-	for i = #entries, 1, -1 do -- Feeds are usually in reverse order
+
+	for i = start_from, 1, -1 do -- Feeds are usually in reverse order
 		local entry = entries[i];
 		entry.attr.xmlns = xmlns_atom;
 
-		local e_published = entry:get_child_text("published");
-		e_published = e_published and dt_parse(e_published);
-		local e_updated = entry:get_child_text("updated");
-		e_updated = e_updated and dt_parse(e_updated);
+		local id = entry:get_child_text("id");
 
-		local timestamp = e_updated or e_published or nil;
-		--module:log("debug", "timestamp is %s, item.last_update is %s", tostring(timestamp), tostring(item.last_update));
+		local timestamp = dt_parse(entry:get_child_text("published"));
+		if not timestamp then
+			timestamp = time();
+			entry:text_tag("published", dt_datetime(timestamp));
+		end
+
 		if not timestamp or not item.last_update or timestamp > item.last_update then
-			local id = entry:get_child_text("id");
-			if not id then
-				local link = entry:get_child("link");
-				id = link and link.attr.href;
-			end
-			if not id then
-				-- Sigh, no link?
-				id = feed.url .. "#" .. hmac_sha1(feed.url, tostring(entry), true) .. "@" .. dt_datetime(timestamp);
-			end
-			if not items[id] then
-				local xitem = st.stanza("item", { id = id, xmlns = "http://jabber.org/protocol/pubsub" }):add_child(entry);
-				-- TODO Put data from /feed into item/source
+			local xitem = st.stanza("item", { id = id, xmlns = "http://jabber.org/protocol/pubsub" }):add_child(entry);
+			-- TODO Put data from /feed into item/source
 
-				--module:log("debug", "publishing to %s, id %s", node, id);
-				local ok, err = pubsub.service:publish(node, true, id, xitem);
-				if not ok then
-					module:log("error", "Publishing to node %s failed: %s", node, err);
-				end
+			local ok, err = pubsub.service:publish(node, true, id, xitem);
+			if not ok then
+				module:log("error", "Publishing to node %s failed: %s", node, err);
+			elseif timestamp then
+				item.last_update = timestamp;
 			end
 		end
 	end
@@ -148,20 +151,18 @@
 end
 
 function fetch(item, callback) -- HTTP Pull
-	local headers = { };
-	if item.data and item.etag then
-		headers["If-None-Match"] = item.etag;
-	end
+	local headers = {
+		["If-None-Match"] = item.etag;
+		["Accept"] = "application/atom+xml, application/x-rss+xml, application/xml";
+	};
 	http.request(item.url, { headers = headers }, function(data, code, resp)
 		if code == 200 then
-			item.data = data;
-			if callback then callback(item) end
-			item.last_update = time();
+			if callback then callback(item, data) end
 			if resp.headers then
 				item.etag = resp.headers.etag
 			end
 		elseif code == 304 then
-			item.last_update = time();
+			module:log("debug", "No updates to %q", item.url);
 		elseif code == 301 and resp.headers.location then
 			module:log("info", "Feed %q has moved to %q", item.url, resp.headers.location);
 		elseif code <= 100 then
@@ -268,9 +269,7 @@
 				end
 				module:log("debug", "Valid signature");
 			end
-			feed.data = body;
-			update_entry(feed);
-			feed.last_update = time();
+			update_entry(feed, body);
 			return 202;
 		end
 		return 400;