diff --git a/Gemfile b/Gemfile index 8f9202c..5b2ba2a 100644 --- a/Gemfile +++ b/Gemfile @@ -62,10 +62,11 @@ group :plugins, :autodiscovery, optional: true do end # The plugins below are Supported (external): each needs a service or a command -# the operator provides, and their specs exercise it rather than a double. They -# are deliberately outside the `plugins` group, so that installing that group -# leaves the suite runnable with nothing else set up. Select one of these by -# its own name when you have what it talks to. +# the operator provides for real use. They are deliberately outside the +# `plugins` group, because installing all of their client gems is not useful +# without those external systems. Their specs verify the local side of each +# integration with doubles where appropriate. Select one by its own name when +# you have what it talks to. group :memcached, optional: true do gem 'dalli' # PublishMemcached, with a memcached server end diff --git a/doc/PLUGINS.md b/doc/PLUGINS.md index f785e83..57d1a11 100644 --- a/doc/PLUGINS.md +++ b/doc/PLUGINS.md @@ -1577,12 +1577,14 @@ headline edited between runs no longer republishes the article. #### StoreFile — **Supported** -`store/file.rb`. Downloads what each link points at and rewrites the link to a -`file://` URI, which is how `PublishAmazonS3` later knows it has a local file. +`store/file.rb`. Downloads what each link points at and rewrites the link to an +absolute `file:` URI for the saved file, which is how `PublishAmazonS3` later +knows it has a local file. URI-reserved characters in the saved path are +percent-encoded. | Key | Type | Meaning | | --- | --- | --- | -| `path` | string | Directory to save into; created if absent. Required. | +| `path` | string | Directory to save into; relative paths use the process working directory; created if absent. Required. | | `retry` | integer | Attempts after the first. Default `0`. | | `interval` | integer | Seconds between downloads. Default `0`. | | `access_key` | string | S3 only. Omit to use the SDK's own credential chain. | @@ -1881,8 +1883,9 @@ timeout, so an unanswered request ends rather than hanging a `cron` job. #### PublishAmazonS3 — **Supported (external)** -`publish/amazon_s3.rb`. Uploads files whose link is a `file://` URI to S3, -normally after `StoreFile`. +`publish/amazon_s3.rb`. Uploads files whose link is a `file:` URI to S3, +normally after `StoreFile`. A percent-encoded URI path is decoded back to the +local filesystem path before upload. | Key | Type | Meaning | | --- | --- | --- | diff --git a/doc/VERSIONS b/doc/VERSIONS index c171bef..6eceb56 100644 --- a/doc/VERSIONS +++ b/doc/VERSIONS @@ -10,6 +10,7 @@ v26.09 (Release Date: TBD) - Preserve nil-link pipeline items and remove SubscriptionText placeholder title and link values. - Preserve SubscriptionText TSV column positions and ignore empty input rows. +- Preserve local file paths when StoreFile hands file URIs to PublishAmazonS3. v26.08 (2026-08-22) ------------------- diff --git a/lib/automatic/cli.rb b/lib/automatic/cli.rb index eebe7b3..bd24aa4 100644 --- a/lib/automatic/cli.rb +++ b/lib/automatic/cli.rb @@ -5,7 +5,7 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Aug 14, 2026 -# Updated:: Sep 6, 2026 +# Updated:: Sep 9, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. # # Everything that belongs to being a command: option parsing, the subcommands, @@ -208,9 +208,19 @@ def feedparser(argv) require 'automatic/feed_parser' require 'pp' url = argv.shift || missing_argument('feedparser') + validate_fetch_url(url) @stdout.puts Automatic::FeedParser.get_url(url).pretty_inspect end + # Converts only the URL validation failure `Automatic::Http.uri` already + # applies into the CLI's ordinary failure path; an unexpected error from + # inside FeedParser itself is left to propagate. + def validate_fetch_url(url) + Automatic::Http.uri(url) + rescue ArgumentError, URI::InvalidURIError => e + raise Automatic::Error, e.message + end + def inspect_url(argv) require 'automatic/feed_parser' Automatic.require_optional('feedbag', needed_by: 'the inspect subcommand') diff --git a/plugins/filter/join.rb b/plugins/filter/join.rb index 739a0a9..4a99a95 100644 --- a/plugins/filter/join.rb +++ b/plugins/filter/join.rb @@ -87,11 +87,9 @@ def value(item, name) item.public_send(name).to_s.strip end - # Built here rather than through FeedMaker.create_pipeline, which drops an - # item whose link is nil -- and this item's link is nil deliberately. It is - # several articles at once, so there is no page it points at, and putting - # the first article's URL there would name a source for text that is not - # only from it. + # This joined item deliberately has no link. It is several articles at once, + # so there is no single page it points at; using the first article's URL would + # name a source for text that is not only from it. def feed(item_title, item_description) RSS::Maker.make('2.0') { |maker| maker.channel.title = 'Automatic Ruby' diff --git a/plugins/publish/amazon_s3.rb b/plugins/publish/amazon_s3.rb index 616948f..d348b64 100644 --- a/plugins/publish/amazon_s3.rb +++ b/plugins/publish/amazon_s3.rb @@ -5,13 +5,15 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 24, 2014 -# Updated:: Aug 15, 2026 +# Updated:: Sep 9, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. module Automatic::Plugin class PublishAmazonS3 require 'uri' + FILE_URI_ESCAPER = URI::RFC2396_Parser.new + def initialize(config, pipeline = []) @config = config || {} @pipeline = pipeline @@ -35,7 +37,7 @@ def publish(feed) uri = URI.parse(feed.link) if uri.scheme == 'file' - upload(uri.path) + upload(file_path(uri)) else Automatic::Log.puts('warn', 'Skip feed due to uri scheme is not file.') end @@ -44,6 +46,10 @@ def publish(feed) "Error detected with #{feed.link} in uploading AmazonS3: #{e.message}") end + def file_path(uri) + FILE_URI_ESCAPER.unescape(uri.path.to_s) + end + def upload(path) key = target_key(path) File.open(path, 'rb') { |body| s3.put_object(bucket: bucket, key: key, body: body) } unless test? diff --git a/plugins/store/file.rb b/plugins/store/file.rb index 8456fe0..6624817 100644 --- a/plugins/store/file.rb +++ b/plugins/store/file.rb @@ -5,7 +5,7 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 28, 2012 -# Updated:: Aug 15, 2026 +# Updated:: Sep 9, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require 'fileutils' @@ -18,6 +18,8 @@ class StoreFile # everything else uses and is accepted as well. S3_SCHEMES = %w[s3 s3n].freeze + FILE_URI_ESCAPER = URI::RFC2396_Parser.new + def initialize(config, pipeline = []) @config = config || {} @pipeline = pipeline @@ -69,7 +71,13 @@ def get_file(url) uri = URI.parse(url) path = S3_SCHEMES.include?(uri.scheme) ? from_s3(uri) : download(url) Automatic::Log.puts('info', "Saved File: #{path}") - "file://#{path}" + file_uri(path) + end + + def file_uri(path) + absolute_path = File.absolute_path(path.to_s) + escaped_path = FILE_URI_ESCAPER.escape(absolute_path) + URI::Generic.build(scheme: 'file', path: escaped_path).to_s end # Only HTTP and HTTPS are fetched: a link arrives from a feed, which is to diff --git a/spec/lib/automatic/cli_spec.rb b/spec/lib/automatic/cli_spec.rb index 7cb2f55..7edea2e 100644 --- a/spec/lib/automatic/cli_spec.rb +++ b/spec/lib/automatic/cli_spec.rb @@ -5,7 +5,7 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Aug 14, 2026 -# Updated:: Sep 6, 2026 +# Updated:: Sep 9, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.join(File.dirname(__FILE__), '../../spec_helper')) @@ -80,6 +80,50 @@ def run(*argv, root_dir: APP_ROOT) end end + describe "the feedparser subcommand" do + it "parses a valid HTTP or HTTPS URL" do + allow(Automatic::FeedParser).to receive(:get_url).and_return("parsed") + + expect(run("feedparser", "https://example.com/feed")).to eq Automatic::CLI::EXIT_SUCCESS + expect(Automatic::FeedParser).to have_received(:get_url).with("https://example.com/feed") + expect(out.string).to match(/parsed/) + expect(err.string).to be_empty + end + + it "rejects a URL whose scheme is not HTTP or HTTPS" do + expect(Automatic::FeedParser).not_to receive(:get_url) + + expect(run("feedparser", "file:///etc/passwd")).to eq Automatic::CLI::EXIT_FAILURE + expect(out.string).to be_empty + expect(err.string).to match(/automatic: not an HTTP or HTTPS URL:/) + end + + it "rejects an HTTP or HTTPS URL with no host" do + expect(Automatic::FeedParser).not_to receive(:get_url) + + expect(run("feedparser", "https:/feed")).to eq Automatic::CLI::EXIT_FAILURE + expect(out.string).to be_empty + expect(err.string).to match(/automatic: HTTP or HTTPS URL has no host:/) + end + + it "rejects a URL that fails to parse as a URI syntax error" do + expect(Automatic::FeedParser).not_to receive(:get_url) + + expect(run("feedparser", "http://[")).to eq Automatic::CLI::EXIT_FAILURE + expect(out.string).to be_empty + expect(err.string).to match(/\Aautomatic: /) + end + + it "propagates an unexpected internal ArgumentError from FeedParser, unconverted" do + allow(Automatic::FeedParser).to receive(:get_url) + .and_raise(ArgumentError, 'internal feed parser defect') + + expect { + run("feedparser", "https://example.com/feed") + }.to raise_error(ArgumentError, /internal feed parser defect/) + end + end + describe "the inspect subcommand" do it "fails cleanly when no feed is discovered" do stub_const("Feedbag", Class.new) diff --git a/spec/plugins/publish/amazon_s3_spec.rb b/spec/plugins/publish/amazon_s3_spec.rb index 3543320..00a8e59 100644 --- a/spec/plugins/publish/amazon_s3_spec.rb +++ b/spec/plugins/publish/amazon_s3_spec.rb @@ -5,7 +5,7 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Feb 25, 2014 -# Updated:: Aug 15, 2026 +# Updated:: Sep 9, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -63,6 +63,25 @@ end end + context 'with a percent-encoded file URI' do + it 'decodes the URI path back to the local filesystem path before uploading' do + Dir.mktmpdir do |dir| + path = File.join(dir, 'a photo.png') + File.binwrite(path, 'x') + + escaped_path = URI::RFC2396_Parser.new.escape(path) + link = URI::Generic.build(scheme: 'file', path: escaped_path).to_s + + plugin = Automatic::Plugin::PublishAmazonS3.new( + settings, + AutomaticSpec.generate_pipeline { feed { item link } } + ) + plugin.should_receive(:upload).with(path).and_call_original + plugin.run.should have(1).feed + end + end + end + describe 'the client settings' do it 'passes the Recipe credentials through' do plugin = Automatic::Plugin::PublishAmazonS3.new(settings.merge('region' => 'ap-northeast-1')) diff --git a/spec/plugins/store/file_spec.rb b/spec/plugins/store/file_spec.rb index 51b5859..ccd9516 100644 --- a/spec/plugins/store/file_spec.rb +++ b/spec/plugins/store/file_spec.rb @@ -5,7 +5,7 @@ # License:: The GPL version 3, or LGPL version 3 (Dual License). # Contact:: idnanashi@gmail.com # Created:: Mar 4, 2012 -# Updated:: Aug 14, 2026 +# Updated:: Sep 9, 2026 # Copyright:: Copyright (c) 2012-2026 Automatic Ruby Developers. require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper') @@ -88,6 +88,30 @@ end end + it "rewrites a relative, space-containing path to an absolute, escaped file URI" do + relative_dir = "automatic-store-file-spec #{Process.pid}" + FileUtils.mkdir_p(relative_dir) + begin + Automatic::Http.stub(:read).and_return('a body') + instance = Automatic::Plugin::StoreFile.new( + { "path" => relative_dir }, + AutomaticSpec.generate_pipeline { feed { item "https://example.com/a/photo.png" } } + ) + + returned = instance.run + saved_path = File.join(relative_dir, 'photo.png') + uri = URI.parse(returned[0].items[0].link) + + uri.scheme.should == 'file' + uri.host.to_s.should be_empty + uri.to_s.should include('%20') + URI::RFC2396_Parser.new.unescape(uri.path).should == File.absolute_path(saved_path) + File.read(saved_path).should == 'a body' + ensure + FileUtils.rm_rf(relative_dir) + end + end + # `s3n` is what Recipes written for this plugin use; `s3` is the spelling # everything else uses and is accepted as well. Both go to the SDK rather # than over HTTP.