diff --git a/.gitignore b/.gitignore index 572ac21ba..dd6458da5 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ tess.crontab app-secrets* node_modules/ + +# Python (test/signposting) +__pycache__/ +*.pyc diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index ce3223022..ebbdf5a61 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -21,13 +21,13 @@ class ApplicationController < ActionController::Base protect_from_forgery with: :exception # Should allow token authentication for API calls - acts_as_token_authentication_handler_for User, except: [:index, :show, :embed, :calendar, :check_exists, :handle_error, :count, + acts_as_token_authentication_handler_for User, except: [:index, :show, :embed, :calendar, :check_exists, :handle_error, :count, :ro_crate, :redirect] #only: [:new, :create, :edit, :update, :destroy] # User auth should be required in the web interface as well; it's here rather than in routes so that it # doesn't override the token auth, above. before_action :set_current_user - before_action :authenticate_user!, except: [:index, :show, :embed, :calendar, :check_exists, :handle_error, :count, :redirect] + before_action :authenticate_user!, except: [:index, :show, :embed, :calendar, :check_exists, :handle_error, :count, :ro_crate, :redirect] before_action :set_current_space # Should prevent forgery errors for JSON posts. diff --git a/app/controllers/concerns/fair_signposting.rb b/app/controllers/concerns/fair_signposting.rb new file mode 100644 index 000000000..02c86a35f --- /dev/null +++ b/app/controllers/concerns/fair_signposting.rb @@ -0,0 +1,105 @@ +# Adds FAIR Signposting (https://signposting.org/FAIR/) links to the show page of a resource, as an HTTP Link header +# and as elements in the HTML head, and serves a Detached RO-Crate of the resource on request. +# +# Including controllers must define +fair_signposting_resource+, returning the resource being shown. +module FairSignposting + extend ActiveSupport::Concern + + SCHEMA_ORG = 'https://schema.org/'.freeze + JSON_LD = 'application/ld+json'.freeze + + included do + helper_method :fair_signposting_links + after_action :set_fair_signposting_header, only: :show, if: -> { response.successful? } + end + + # GET /materials/1/ro-crate-metadata.json + def ro_crate + resource = fair_signposting_resource + authorize resource, :show? + render_ro_crate(resource) + end + + private + + # Whether the RO-Crate is requested by content negotiation for JSON-LD with the RO-Crate profile, e.g. + # "Accept: application/ld+json; profile="https://w3id.org/ro/crate"". + def ro_crate_requested? + profile = Regexp.escape(RoCrateExport::DetachedCrate::PROFILE) + request.headers['Accept'].to_s.match?(%r{application/ld\+json\s*;[^,]*profile="?#{profile}(?=["\s,;]|\z)}i) + end + + def render_ro_crate(resource) + render plain: RoCrateExport::DetachedCrate.new(resource).to_json, + content_type: RoCrateExport::DetachedCrate::MEDIA_TYPE + end + + # An array of [url, { attribute => value }] pairs. + def fair_signposting_links + resource = fair_signposting_resource + return [] unless resource&.persisted? + + @fair_signposting_links ||= described_by_links(resource) + cite_as_links(resource) + author_links(resource) + + licence_links(resource) + type_links(resource) + end + + def described_by_links(resource) + [ + [polymorphic_url(resource, format: :jsonld), + { rel: 'describedby', type: JSON_LD, profile: bioschemas_generator(resource).bioschemas_profile }.compact], + [polymorphic_url([:ro_crate_metadata, resource]), + { rel: 'describedby', type: JSON_LD, profile: RoCrateExport::DetachedCrate::PROFILE }], + [polymorphic_url(resource, format: :json_api), { rel: 'describedby', type: 'application/vnd.api+json' }] + ] + end + + def cite_as_links(resource) + doi_url = RoCrateExport::Identifier.url(resource.try(:doi)) + doi_url ? [[doi_url, { rel: 'cite-as' }]] : [] + end + + def author_links(resource) + Array(resource.try(:authors)).filter_map(&:orcid_url).uniq.map { |orcid| [orcid, { rel: 'author' }] } + end + + def licence_links(resource) + licence_url = RoCrateExport::Identifier.licence_url(resource.try(:licence)) + licence_url ? [[licence_url, { rel: 'license' }]] : [] + end + + def type_links(resource) + ["#{SCHEMA_ORG}#{bioschemas_generator(resource).type}", "#{SCHEMA_ORG}AboutPage"].map { |t| [t, { rel: 'type' }] } + end + + def bioschemas_generator(resource) + resource.to_bioschemas.first.class + end + + def set_fair_signposting_header + links = fair_signposting_links + return if links.empty? + + signposting = links.map do |url, attributes| + (["<#{url}>"] + attributes.map { |key, value| %(#{key}="#{value}") }).join(' ; ') + end + # Keep any existing links, e.g. asset preload hints added by Rails, but after the signposting links: some clients, + # such as the Signposting Sniffing browser extension, ignore the whole header unless its first link is a signpost. + existing = response.get_header('Link').presence + response.set_header('Link', [*signposting, existing && quote_link_values(existing)].compact.join(', ')) + end + + # Rails' asset preload links have unquoted attribute values that are not valid RFC 8288 tokens (e.g. base64 + # integrity hashes ending in "="), and a valueless "nopush" (a hint for the obsolete HTTP/2 server push), which + # make strict Link header parsers reject the whole header, including the signposting links. Quote the values and + # drop "nopush". + def quote_link_values(header) + header.split(/,\s*(?=<)/).map do |link| + target, params = link.split(/(?<=>)/, 2) + params = params.to_s.gsub(/;\s*nopush\s*(?=;|\z)/, '') + params = params.gsub(/;\s*(?[\w*-]+)=(?[^";,]+?)\s*(?=;|\z)/) do + %(; #{Regexp.last_match[:key]}="#{Regexp.last_match[:value]}") + end + "#{target}#{params}" + end.join(', ') + end +end diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index bdfe153fb..f40fc1a16 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -3,7 +3,7 @@ # The controller for actions related to the Events model class EventsController < ApplicationController before_action :ensure_feature_enabled - before_action :set_event, only: %i[show edit clone update destroy update_collections add_term reject_term + before_action :set_event, only: %i[show ro_crate edit clone update destroy update_collections add_term reject_term redirect report update_report add_data reject_data] before_action :set_breadcrumbs before_action :disable_pagination, only: :index, if: ->(controller) { controller.request.format.ics? or controller.request.format.csv? or controller.request.format.rss? } @@ -12,6 +12,7 @@ class EventsController < ApplicationController include ActionView::Helpers::TextHelper include FieldLockEnforcement include TopicCuration + include FairSignposting # GET /events # GET /events.json @@ -73,7 +74,9 @@ def show format.json format.json_api { render json: @event } format.ics { send_data @event.to_ical, type: 'text/calendar', disposition: 'attachment', filename: "#{@event.slug}.ics" } - format.jsonld { render plain: @bioschemas.first.to_json } + format.jsonld do + ro_crate_requested? ? render_ro_crate(@event) : render(plain: @bioschemas.first.to_json) + end end end @@ -228,6 +231,10 @@ def redirect private + def fair_signposting_resource + @event if %w[show ro_crate].include?(action_name) + end + # Use callbacks to share common setup or constraints between actions. def set_event @event = Event.friendly.find(params[:id]) diff --git a/app/controllers/materials_controller.rb b/app/controllers/materials_controller.rb index 9a53fa0d3..67f8f8bf0 100644 --- a/app/controllers/materials_controller.rb +++ b/app/controllers/materials_controller.rb @@ -1,7 +1,7 @@ # The controller for actions related to the Materials model class MaterialsController < ApplicationController before_action :ensure_feature_enabled - before_action :set_material, only: %i[show edit update destroy update_collections clone + before_action :set_material, only: %i[show ro_crate edit update destroy update_collections clone add_term reject_term add_data reject_data] before_action :set_breadcrumbs before_action :set_learning_path_navigation, only: :show @@ -10,6 +10,7 @@ class MaterialsController < ApplicationController include ActionView::Helpers::TextHelper include FieldLockEnforcement include TopicCuration + include FairSignposting # GET /materials # GET /materials?q=queryparam @@ -35,7 +36,9 @@ def show format.html format.json format.json_api { render json: @material } - format.jsonld { render plain: @bioschemas.first.to_json } + format.jsonld do + ro_crate_requested? ? render_ro_crate(@material) : render(plain: @bioschemas.first.to_json) + end end end @@ -158,6 +161,10 @@ def update_collections private + def fair_signposting_resource + @material if %w[show ro_crate].include?(action_name) + end + # Use callbacks to share common setup or constraints between actions. def set_material @material = Material.friendly.find(params[:id]) diff --git a/app/views/about/developers.erb b/app/views/about/developers.erb index 9ee40c453..bc0dd28a6 100644 --- a/app/views/about/developers.erb +++ b/app/views/about/developers.erb @@ -112,6 +112,35 @@ <% end %>

+ <% if feature_enabled?('events') || feature_enabled?('materials') %> +

Bioschemas, RO-Crate and FAIR Signposting

+

+ The pages of events and training materials embed + Bioschemas + JSON-LD, which can also be retrieved by appending .jsonld to the URL. Each event and material + is also available as a + Detached RO-Crate, + with the <%= TeSS::Config.site['title_short'] %> record as its root and the event or material itself as + its main entity. For example: +

+

+ <% resource = feature_enabled?('materials') ? 'materials' : 'events' %> + <%= TeSS::Config.base_url %>/<%= resource %>/<id>.jsonld
+ <%= TeSS::Config.base_url %>/<%= resource %>/<id>/ro-crate-metadata.json
+ curl -H 'Accept: application/ld+json; profile="https://w3id.org/ro/crate"' <%= TeSS::Config.base_url %>/<%= resource %>/<id> +

+

+ These representations, and the DOI (cite-as), authors' ORCIDs (author), licence + (license) and type (type) of the resource, are advertised by + FAIR Signposting + links (describedby) in the HTTP Link header and the HTML + <link> elements of each page, so that clients can find them: +

+

+ curl -I <%= TeSS::Config.base_url %>/<%= resource %>/<id> +

+ <% end %> +

We recommend using the JSON API, but if you have already begun using our old API, technical information can still be found here: diff --git a/app/views/layouts/_head.html.erb b/app/views/layouts/_head.html.erb index 5b83760d8..9e199f87f 100644 --- a/app/views/layouts/_head.html.erb +++ b/app/views/layouts/_head.html.erb @@ -33,5 +33,11 @@ <% end %> <% end %> + <% if respond_to?(:fair_signposting_links) %> + <% fair_signposting_links.each do |url, attributes| %> + <%= tag.link(href: url, **attributes) %> + <% end %> + <% end %> + <%= yield :extra_head if content_for? :extra_head %> diff --git a/config/routes.rb b/config/routes.rb index 627892161..b204898f5 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -76,6 +76,7 @@ get :report patch :report, to: 'events#update_report' get :clone + get 'ro-crate-metadata.json', action: :ro_crate, as: :ro_crate_metadata, format: false end end @@ -114,6 +115,7 @@ post :add_term post :add_data get :clone + get 'ro-crate-metadata.json', action: :ro_crate, as: :ro_crate_metadata, format: false end collection do get :count diff --git a/docs/api.md b/docs/api.md index 5980765a9..a49e1eedb 100644 --- a/docs/api.md +++ b/docs/api.md @@ -38,6 +38,30 @@ A record can be viewed through this API by appending `.json_api` to the URL, for http://localhost:3000/materials.json_api http://localhost:3000/materials/1.json_api +## Bioschemas, RO-Crate and FAIR Signposting + +The pages of Materials and Events embed [Bioschemas](https://bioschemas.org/) JSON-LD, which can also be retrieved by +appending `.jsonld` to the URL: + + http://localhost:3000/materials/1.jsonld + +Each Material and Event is also available as a [Detached RO-Crate](https://w3id.org/ro/crate/1.3), with the record's +URL as the Root Data Entity, a `Dataset`. The Material or Event itself is its `mainEntity`: the Bioschemas +`LearningResource`, `Event` or `Course`, identified by its external URL. A Material is also a part of the crate +(`hasPart`). Licences are given as SPDX license URIs, e.g. `http://spdx.org/licenses/CC-BY-4.0`: + + http://localhost:3000/materials/1/ro-crate-metadata.json + curl -H 'Accept: application/ld+json; profile="https://w3id.org/ro/crate"' http://localhost:3000/materials/1 + +These representations are advertised by [FAIR Signposting](https://signposting.org/FAIR/) links, in the HTTP `Link` +header and as `` elements in the HTML head of the page: `describedby` for the Bioschemas JSON-LD, RO-Crate and +JSON-API, `cite-as` for the DOI (as an `https://doi.org/` URL), `author` for authors' ORCIDs, `license` and `type`. + + curl -I http://localhost:3000/materials/1 + +Integration tests that check these with the [signposting](https://pypi.org/project/signposting/) library against a +running TeSS are in [test/signposting](../test/signposting/test_signposting.py). + ## Legacy API A simple read/write API supporting Events and Materials. diff --git a/lib/ro_crate_export/detached_crate.rb b/lib/ro_crate_export/detached_crate.rb new file mode 100644 index 000000000..2a958f597 --- /dev/null +++ b/lib/ro_crate_export/detached_crate.rb @@ -0,0 +1,153 @@ +module RoCrateExport + # Builds a Detached RO-Crate (https://w3id.org/ro/crate/1.3) describing a single TeSS resource. + # The Root Data Entity is a Dataset for the TeSS record, with the TeSS page URL as its absolute @id. Its + # +mainEntity+ is the resource's Bioschemas representation (e.g. a LearningResource or Event), flattened into the + # RO-Crate @graph with the resource's external URL as its @id. A material is also a part (+hasPart+) of the crate; + # an event is not a data entity, so it is only the main entity. + class DetachedCrate + SPEC = 'https://w3id.org/ro/crate/1.3'.freeze + CONTEXT = "#{SPEC}/context".freeze + PROFILE = 'https://w3id.org/ro/crate'.freeze + MEDIA_TYPE = %(application/ld+json; profile="#{PROFILE}").freeze + METADATA_ID = 'ro-crate-metadata.json'.freeze + + # Bioschemas generator properties that are not defined by the RO-Crate JSON-LD context. + UNDEFINED_PROPERTIES = %w[contact eligibility event_types hostInstitution].freeze + + NO_LICENCE = 'No licence has been specified for this resource in TeSS.'.freeze + + # Resources that are data entities of the crate, not only its main entity. + PART_CLASSES = %w[Material].freeze + + attr_reader :resource + + def initialize(resource) + @resource = resource + @bioschemas = resource.to_bioschemas.first + end + + def generate + @entities = {} + @local_ids = Hash.new(0) + bioschemas = @bioschemas.generate.transform_keys(&:to_s) + main = main_entity(bioschemas) + main_ref = add_entity(main) + root = add_entity(root_data_entity(bioschemas['@id'], main, main_ref)) + # List the Root Data Entity first after the descriptor, for readers that do not look it up by @id + @entities = { root['@id'] => @entities[root['@id']] }.merge(@entities) + + descriptor = { + '@id' => METADATA_ID, + '@type' => 'CreativeWork', + 'about' => root, + 'conformsTo' => { '@id' => SPEC } + } + + { '@context' => CONTEXT, '@graph' => [descriptor, *@entities.values] } + end + + def to_json(*_args) + JSON.pretty_generate(generate) + end + + private + + # The TeSS record, with the properties required of a Root Data Entity. + def root_data_entity(page_url, main, main_ref) + { + '@id' => page_url, + '@type' => 'Dataset', + 'name' => main['name'], + 'description' => main['description'], + 'datePublished' => date_published, + 'license' => licence || NO_LICENCE, + 'publisher' => publisher, + 'mainEntity' => main_ref, + 'hasPart' => (main_ref if PART_CLASSES.include?(resource.class.name) && main['@id'].start_with?('http')) + } + end + + # The resource's Bioschemas properties, identified by its external URL (or a local identifier if it has none). + # Its Bioschemas +url+ is a plain string, so it is replaced by the @id. + def main_entity(bioschemas) + main = bioschemas.except('@context', 'dct:conformsTo', *UNDEFINED_PROPERTIES) + url = external_url + main.merge( + '@id' => url || local_id(main['@type']), + 'url' => (main['url'] unless url), + 'identifier' => Identifier.url(main['identifier']), + 'description' => main['description'].presence || main['name'], + 'license' => licence || main['license'], + 'conformsTo' => profile, + 'sdPublisher' => publisher + ) + end + + def external_url + url = resource.try(:url).to_s.strip + url if url.match?(%r{\Ahttps?://[^\s/]+\.[^\s/]+(/\S*)?\z}i) + end + + def date_published + date = resource.try(:date_published) || resource.created_at || Time.now + date.iso8601 + end + + def licence + url = Identifier.licence_url(resource.try(:licence)) + return nil unless url + + title = LicenceDictionary.instance.lookup_value(resource.licence, 'title') + { '@id' => url, '@type' => 'CreativeWork', 'name' => title || url } + end + + def profile + return nil unless @bioschemas.class.bioschemas_profile + + { '@id' => @bioschemas.class.bioschemas_profile, '@type' => 'Profile', + 'name' => "Bioschemas #{@bioschemas.class.type} profile" } + end + + def publisher + url = Rails.application.routes.url_helpers.root_url + { '@id' => url, '@type' => 'Organization', 'name' => TeSS::Config.site['title'], 'url' => url } + end + + # Add an entity to the graph, replacing nested objects with references to flattened entities. + def add_entity(entity) + entity = flatten_properties(entity) + existing = @entities[entity['@id']] + @entities[entity['@id']] = existing ? existing.merge(entity) : entity + { '@id' => entity['@id'] } + end + + def flatten_properties(entity) + entity.to_h.transform_keys(&:to_s).each_with_object({}) do |(key, value), flat| + next if value.blank? + + flat[key] = key.start_with?('@') ? value : flatten_value(value) + end + end + + def flatten_value(value) + case value + when Array + values = value.map { |v| flatten_value(v) }.compact + values.length == 1 ? values.first : values + when Hash + nested = value.transform_keys(&:to_s) + return nested if nested.keys == ['@id'] + + nested['@id'] ||= local_id(nested['@type']) + add_entity(nested) + else + value + end + end + + def local_id(type) + name = Array(type).first.to_s.underscore.dasherize.presence || 'entity' + "##{name}-#{@local_ids[name] += 1}" + end + end +end diff --git a/lib/ro_crate_export/identifier.rb b/lib/ro_crate_export/identifier.rb new file mode 100644 index 000000000..bc60f837e --- /dev/null +++ b/lib/ro_crate_export/identifier.rb @@ -0,0 +1,35 @@ +module RoCrateExport + # Normalizes identifiers for FAIR Signposting and RO-Crate. + module Identifier + DOI_RESOLVER = 'https://doi.org/'.freeze + DOI_PREFIX = %r{\A(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)}i + DOI = %r{\A10\.\d{4,}(?:\.\d+)*/\S+\z} + + # Returns a DOI (bare, "doi:" or doi.org URL) as an https://doi.org/ URL, any other http(s) URL unchanged, + # or nil if the value is neither. + def self.url(value) + value = value.to_s.strip + doi = value.sub(DOI_PREFIX, '') + return "#{DOI_RESOLVER}#{doi}" if doi.match?(DOI) + return value if value.match?(%r{\Ahttps?://\S+\z}i) + + nil + end + + SPDX_LICENCES = 'http://spdx.org/licenses/'.freeze + SPDX_REFERENCE = %r{\Ahttps?://spdx\.org/licenses/(?[^/]+?)(?:\.html|\.json)?\z}i + + # Returns the URL of a TeSS licence (e.g. "CC-BY-4.0"): for SPDX licences the SPDX license URI, e.g. + # http://spdx.org/licenses/CC-BY-4.0, which matches SPDX's RDF, rather than the HTML page in the licence + # dictionary; otherwise the dictionary's reference URL, or nil if there is none. + def self.licence_url(licence) + return nil if licence.blank? + + reference = LicenceDictionary.instance.lookup_value(licence, 'reference').presence + spdx = reference&.match(SPDX_REFERENCE) + return "#{SPDX_LICENCES}#{spdx[:id]}" if spdx + + reference if reference&.match?(%r{\Ahttps?://\S+\z}i) + end + end +end diff --git a/test/controllers/about_controller_test.rb b/test/controllers/about_controller_test.rb index 6f550d878..f58eb4f41 100644 --- a/test/controllers/about_controller_test.rb +++ b/test/controllers/about_controller_test.rb @@ -106,6 +106,9 @@ class AboutControllerTest < ActionController::TestCase test 'should get about developers' do get :developers assert_response :success + assert_select '#api #fair-signposting', text: 'Bioschemas, RO-Crate and FAIR Signposting' + assert_select '#api a[href="https://signposting.org/FAIR/"]' + assert_select '#api code', text: %r{/ro-crate-metadata\.json} end test 'should get about learning paths' do diff --git a/test/integration/fair_signposting_test.rb b/test/integration/fair_signposting_test.rb new file mode 100644 index 000000000..98a9813b2 --- /dev/null +++ b/test/integration/fair_signposting_test.rb @@ -0,0 +1,229 @@ +require 'test_helper' + +class FairSignpostingTest < ActionDispatch::IntegrationTest + RO_CRATE_PROFILE = 'https://w3id.org/ro/crate'.freeze + SIGNPOSTING_RELS = %w[describedby cite-as author license type item].freeze + + test 'fair signposting for material' do + material = materials(:good_material) + material.authors.create!(name: 'Ada Lovelace', orcid: '0000-0002-1825-0097') + material.authors.create!(name: 'Charles Babbage') + + get material_path(material) + + assert_response :success + assert_match(/rel="preload"/, response.headers['Link'], 'Rails preload links should be kept') + assert_match(/\A<[^>]*>\s*;\s*rel="describedby"/, response.headers['Link'], 'Signposting links should come first') + links = parse_link_header + assert_equal 8, links.size + assert_link(links, material_url(material, format: :jsonld), rel: 'describedby', type: 'application/ld+json', + profile: 'https://bioschemas.org/profiles/TrainingMaterial/1.0-RELEASE') + assert_link(links, ro_crate_metadata_material_url(material), rel: 'describedby', type: 'application/ld+json', + profile: RO_CRATE_PROFILE) + assert_link(links, material_url(material, format: :json_api), rel: 'describedby', type: 'application/vnd.api+json') + assert_link(links, 'http://dx.doi.ac.uk/2q3093032', rel: 'cite-as') + assert_link(links, 'https://orcid.org/0000-0002-1825-0097', rel: 'author') + assert_link(links, 'http://spdx.org/licenses/CC-BY-4.0', rel: 'license') + assert_link(links, 'https://schema.org/LearningResource', rel: 'type') + assert_link(links, 'https://schema.org/AboutPage', rel: 'type') + + html_links = Nokogiri::HTML(response.body).css('head link[rel="describedby"]').map { |l| l['href'] } + assert_includes html_links, ro_crate_metadata_material_url(material) + end + + test 'fair signposting for material without doi, licence or authors' do + material = materials(:bad_material) + + get material_path(material) + + assert_response :success + links = parse_link_header + assert_equal 5, links.size + assert_empty(links.select { |l| %w[cite-as author license].include?(l[:rel]) }) + end + + test 'doi rewritten to https://doi.org/ in cite-as and ro-crate' do + material = materials(:biojs) + material.update_column(:doi, 'http://dx.doi.org/10.1109/SEAA.2009.59') + + get material_path(material) + + assert_response :success + assert_link(parse_link_header, 'https://doi.org/10.1109/SEAA.2009.59', rel: 'cite-as') + + get ro_crate_metadata_material_path(material) + + main = JSON.parse(response.body)['@graph'].detect { |e| e['@id'] == material.url } + assert_equal 'https://doi.org/10.1109/SEAA.2009.59', main['identifier'] + end + + test 'fair signposting for event' do + event = events(:one) + + get event_path(event) + + assert_response :success + links = parse_link_header + assert_equal 5, links.size + assert_link(links, event_url(event, format: :jsonld), rel: 'describedby', type: 'application/ld+json', + profile: 'https://bioschemas.org/profiles/Course/1.0-RELEASE') + assert_link(links, ro_crate_metadata_event_url(event), rel: 'describedby', type: 'application/ld+json', + profile: RO_CRATE_PROFILE) + assert_link(links, 'https://schema.org/Course', rel: 'type') + end + + test 'no fair signposting on index' do + get materials_path + + assert_response :success + assert_empty parse_link_header + end + + test 'detached ro-crate for material' do + material = materials(:good_material) + + get ro_crate_metadata_material_path(material) + + assert_response :success + assert response.content_type.start_with?(%(application/ld+json; profile="#{RO_CRATE_PROFILE}")) + crate = JSON.parse(response.body) + assert_equal 'https://w3id.org/ro/crate/1.3/context', crate['@context'] + entities = crate['@graph'].index_by { |e| e['@id'] } + + descriptor = entities['ro-crate-metadata.json'] + assert_equal 'https://w3id.org/ro/crate/1.3', descriptor.dig('conformsTo', '@id') + assert_equal material_url(material), descriptor.dig('about', '@id') + + root = entities[material_url(material)] + assert_equal crate['@graph'][1], root, 'Root Data Entity should follow the metadata descriptor' + assert_equal 'Dataset', root['@type'] + assert_equal 'Training Material Example', root['name'] + assert_equal 'This is a Training Material produced by an example organization', root['description'] + assert root['datePublished'].present? + assert_equal 'http://spdx.org/licenses/CC-BY-4.0', root.dig('license', '@id') + assert_equal 'Creative Commons Attribution 4.0 International', + entities['http://spdx.org/licenses/CC-BY-4.0']['name'] + assert_equal root_url, root.dig('publisher', '@id') + assert_nil root['conformsTo'] + + # The material itself, at its external URL, is the main entity and a part of the crate + assert_equal({ '@id' => 'http://training.com/trainers/1359023' }, root['mainEntity']) + assert_equal({ '@id' => 'http://training.com/trainers/1359023' }, root['hasPart']) + main = entities['http://training.com/trainers/1359023'] + assert_equal 'LearningResource', main['@type'] + assert_equal 'Training Material Example', main['name'] + assert_equal 'This is a Training Material produced by an example organization', main['description'] + assert_nil main['url'] + assert_equal 'http://spdx.org/licenses/CC-BY-4.0', main.dig('license', '@id') + assert_equal 'https://bioschemas.org/profiles/TrainingMaterial/1.0-RELEASE', main.dig('conformsTo', '@id') + assert_equal 'Profile', entities['https://bioschemas.org/profiles/TrainingMaterial/1.0-RELEASE']['@type'] + assert_equal root_url, main.dig('sdPublisher', '@id') + assert_nil main['dct:conformsTo'] + assert_nil main['@context'] + assert_equal 1, crate['@graph'].count { |e| e['name'] == 'Goblet' }, 'Provider should be flattened once' + + assert_flattened(crate) + end + + test 'local main entity and no part in detached ro-crate for material without a valid url' do + material = materials(:good_material) + material.update_column(:url, 'Not really a valid URL.com') + + get ro_crate_metadata_material_path(material) + + assert_response :success + crate = JSON.parse(response.body) + entities = crate['@graph'].index_by { |e| e['@id'] } + root = entities[material_url(material)] + assert_nil root['hasPart'] + main = entities[root.dig('mainEntity', '@id')] + assert main['@id'].start_with?('#') + assert_equal 'LearningResource', main['@type'] + assert_equal 'Not really a valid URL.com', main['url'] + assert_flattened(crate) + end + + test 'detached ro-crate for event' do + event = events(:one) + + get ro_crate_metadata_event_path(event) + + assert_response :success + crate = JSON.parse(response.body) + entities = crate['@graph'].index_by { |e| e['@id'] } + root = entities[event_url(event)] + assert_equal 'Dataset', root['@type'] + assert_equal RoCrateExport::DetachedCrate::NO_LICENCE, root['license'] + + # The event is the main entity, but not a data entity + assert_equal({ '@id' => 'http://example.com/cool-event' }, root['mainEntity']) + assert_nil root['hasPart'] + main = entities['http://example.com/cool-event'] + assert_equal 'Course', main['@type'] + assert_equal 'event one', main['name'] + RoCrateExport::DetachedCrate::UNDEFINED_PROPERTIES.each do |property| + assert_nil main[property] + end + + assert_flattened(crate) + end + + test 'detached ro-crate by content negotiation' do + material = materials(:good_material) + + get material_path(material), headers: { 'Accept' => %(application/ld+json; profile="#{RO_CRATE_PROFILE}") } + + assert_response :success + assert_equal 'ro-crate-metadata.json', JSON.parse(response.body)['@graph'].first['@id'] + end + + test 'bioschemas json-ld without ro-crate profile' do + material = materials(:good_material) + + get material_path(material), headers: { 'Accept' => 'application/ld+json' } + + assert_response :success + assert_equal 'LearningResource', JSON.parse(response.body)['@type'] + end + + test 'no ro-crate for material from unverified user' do + material = materials(:good_material) + material.update_column(:user_id, users(:unverified_user).id) + + get ro_crate_metadata_material_path(material) + + assert_response :forbidden + end + + private + + # Signposting links in the Link header, ignoring others such as asset preload hints. + def parse_link_header + links = response.headers['Link'].to_s.split(/,\s*(?=<)/).map { |link| parse_link(link) } + links.select { |link| SIGNPOSTING_RELS.include?(link[:rel]) } + end + + def parse_link(link) + url, *params = link.split(/\s*;\s*/) + params.map { |p| p.split('=', 2) }.to_h { |k, v| [k.to_sym, v&.delete('"')] }.merge(url: url[1..-2]) + end + + def assert_link(links, url, **attributes) + expected = attributes.merge(url:) + assert links.include?(expected), "Expected link #{expected.inspect} in:\n#{links.map(&:inspect).join("\n")}" + end + + # Every entity is at the top level of the @graph, and other entities are only referenced by @id. + def assert_flattened(crate) + ids = crate['@graph'].map { |e| e['@id'] } + assert_equal ids.uniq, ids + crate['@graph'].each do |entity| + assert entity['@type'].present?, "#{entity['@id']} has no @type" + entity.except('@id', '@type').each_value do |value| + Array(value).grep(Hash).each do |ref| + assert_equal ['@id'], ref.keys, "Nested entity in #{entity['@id']}: #{ref.inspect}" + end + end + end + end +end diff --git a/test/signposting/requirements.txt b/test/signposting/requirements.txt new file mode 100644 index 000000000..5d4dcf8ee --- /dev/null +++ b/test/signposting/requirements.txt @@ -0,0 +1,4 @@ +pytest>=8 +requests>=2.31 +signposting>=0.9.9 +roc-validator>=0.13 diff --git a/test/signposting/test_signposting.py b/test/signposting/test_signposting.py new file mode 100644 index 000000000..1968c0cb7 --- /dev/null +++ b/test/signposting/test_signposting.py @@ -0,0 +1,206 @@ +"""FAIR Signposting and RO-Crate integration tests against a running TeSS instance. + +These use the `signposting` library (https://pypi.org/project/signposting/) as an independent client to check the +Link headers and HTML elements of material and event pages, then follow them to the Detached RO-Crate. + +Run against the test fixtures (see README.md), or any TeSS instance by setting TESS_URL: + + RAILS_ENV=test bin/rails db:fixtures:load + RAILS_ENV=test bin/rails server -p 3000 & + pip install -r test/signposting/requirements.txt + TESS_URL=http://localhost:3000 pytest test/signposting +""" +import json +import os +import shutil +import subprocess +from urllib.parse import urljoin, urlsplit + +import pytest +import requests +import signposting + +TESS_URL = os.environ.get("TESS_URL", "http://localhost:3000").rstrip("/") + "/" +RO_CRATE_PROFILE = "https://w3id.org/ro/crate" +RO_CRATE_SPEC = "https://w3id.org/ro/crate/1.3" +JSON_LD = "application/ld+json" +SCHEMA_TYPES = { + "materials": {"https://schema.org/LearningResource"}, + "events": {"https://schema.org/Event", "https://schema.org/Course"}, +} +SAMPLE_SIZE = int(os.environ.get("TESS_SAMPLE_SIZE", "5")) + + +def resource_urls(kind, sample_size=SAMPLE_SIZE): + """Page URLs of the first resources of a kind ("materials" or "events"), found through the JSON:API.""" + response = requests.get(urljoin(TESS_URL, f"{kind}.json_api"), params={"per_page": sample_size}, timeout=30) + response.raise_for_status() + return [urljoin(TESS_URL, item["links"]["self"]) for item in response.json()["data"]] + + +def resources(kind): + try: + urls = resource_urls(kind) + except requests.RequestException as e: + pytest.skip(f"TeSS not reachable at {TESS_URL}: {e}") + if not urls: + pytest.skip(f"No {kind} in TeSS at {TESS_URL}") + return urls + + +def ro_crate_link(signposts): + links = [d for d in signposts.describedBy if RO_CRATE_PROFILE in signposts_profiles(d)] + assert len(links) == 1, f"Expected one RO-Crate describedby link, got {signposts.describedBy}" + return links[0] + + +def signposts_profiles(signpost): + return {str(p) for p in (signpost.profiles or [])} + + +def root_entity(crate): + entities = {e["@id"]: e for e in crate["@graph"]} + descriptor = entities["ro-crate-metadata.json"] + return descriptor, entities[descriptor["about"]["@id"]], entities + + +@pytest.fixture(scope="module", params=["materials", "events"]) +def kind(request): + return request.param + + +@pytest.fixture(scope="module") +def page_urls(kind): + return resources(kind) + + +def test_http_signposting(kind, page_urls): + for url in page_urls: + signposts = signposting.find_signposting_http(url) + + assert signposts.signposts, f"No signposting in Link header of {url}" + link = ro_crate_link(signposts) + assert str(link.type) == JSON_LD + assert str(link.target) == url + "/ro-crate-metadata.json" + + bioschemas = [d for d in signposts.describedBy if str(d.target) == url + ".jsonld"] + assert len(bioschemas) == 1 + assert any(p.startswith("https://bioschemas.org/profiles/") for p in signposts_profiles(bioschemas[0])) + + types = {str(t.target) for t in signposts.types} + assert "https://schema.org/AboutPage" in types + assert types & SCHEMA_TYPES[kind], f"Expected one of {SCHEMA_TYPES[kind]} in {types}" + + if signposts.citeAs: + assert str(signposts.citeAs.target).startswith(("http://", "https://")) + assert not str(signposts.citeAs.target).startswith(("http://doi.org/", "http://dx.doi.org/", + "https://dx.doi.org/")), \ + "DOIs should be given as https://doi.org/ URLs" + + +def test_html_signposting_matches_http(page_urls): + for url in page_urls: + http = signposting.find_signposting_http(url) + html = signposting.find_signposting_html(url) + + def links(s): + return {(str(l.rel), str(l.target), str(l.type or ""), frozenset(signposts_profiles(l))) + for l in s.signposts} + + assert links(html) == links(http) + + +def test_follow_signposting_to_ro_crate(kind, page_urls): + for url in page_urls: + link = ro_crate_link(signposting.find_signposting_http(url)) + response = requests.get(str(link.target), timeout=30) + response.raise_for_status() + assert response.headers["Content-Type"].startswith(f'{JSON_LD}; profile="{RO_CRATE_PROFILE}"') + + crate = response.json() + assert crate["@context"] == f"{RO_CRATE_SPEC}/context" + descriptor, root, entities = root_entity(crate) + assert descriptor["conformsTo"] == {"@id": RO_CRATE_SPEC} + # Detached crate: the Root Data Entity is the page the signposting came from. Its host is TeSS's configured + # base_url, which may differ from the host the tests reach the server through. + assert urlsplit(root["@id"]).path == urlsplit(url).path + assert "Dataset" in root["@type"] + for prop in ("name", "description", "datePublished", "license"): + assert root.get(prop), f"Root Data Entity of {url} has no {prop}" + + # The resource is the main entity, with its Bioschemas type and profile; only materials are data entities + main = entities[root["mainEntity"]["@id"]] + assert "Dataset" not in main["@type"] + assert main["conformsTo"]["@id"].startswith("https://bioschemas.org/profiles/") + if kind == "materials" and not main["@id"].startswith("#"): + assert root["hasPart"] == {"@id": main["@id"]} + if kind == "events": + assert "hasPart" not in root + + # Flattened: nested objects are only references, and every reference is described + for entity in crate["@graph"]: + for key, value in entity.items(): + if key.startswith("@"): + continue + for v in value if isinstance(value, list) else [value]: + if isinstance(v, dict): + assert list(v) == ["@id"], f"Nested entity in {entity['@id']}.{key}: {v}" + if v["@id"].startswith("#"): + assert v["@id"] in entities, f"{v['@id']} is not described" + + +def test_content_negotiation(page_urls): + for url in page_urls: + response = requests.get(url, headers={"Accept": f'{JSON_LD}; profile="{RO_CRATE_PROFILE}"'}, timeout=30) + response.raise_for_status() + assert response.headers["Content-Type"].startswith(JSON_LD) + descriptor, root, _ = root_entity(response.json()) + assert descriptor["conformsTo"] == {"@id": RO_CRATE_SPEC} + + # Without the profile, the Bioschemas JSON-LD is served as before + bioschemas = requests.get(url, headers={"Accept": JSON_LD}, timeout=30).json() + assert "@graph" not in bioschemas + + +def test_doi_rewritten_to_https_doi_org(): + """Materials whose DOI is given as http://doi.org/, http://dx.doi.org/, doi: or bare are cited as https://doi.org/.""" + response = requests.get(urljoin(TESS_URL, "materials.json_api"), params={"per_page": 100}, timeout=30) + response.raise_for_status() + with_doi = [m for m in response.json()["data"] + if (m["attributes"].get("doi") or "").lower().startswith(("10.", "doi:", "http://doi.org/", + "http://dx.doi.org/", + "https://dx.doi.org/"))] + if not with_doi: + pytest.skip("No materials with a DOI to rewrite") + + for material in with_doi: + url = urljoin(TESS_URL, material["links"]["self"]) + doi = material["attributes"]["doi"].strip() + suffix = doi[doi.index("10."):] + signposts = signposting.find_signposting_http(url) + assert str(signposts.citeAs.target) == f"https://doi.org/{suffix}" + + crate = requests.get(str(ro_crate_link(signposts).target), timeout=30).json() + _, root, entities = root_entity(crate) + assert entities[root["mainEntity"]["@id"]]["identifier"] == f"https://doi.org/{suffix}" + + +@pytest.mark.skipif(not shutil.which("rocrate-validator"), reason="rocrate-validator (roc-validator) not installed") +def test_ro_crate_validates(kind, page_urls, tmp_path): + """No REQUIRED issues from rocrate-validator for the RO-Crate 1.3 profile.""" + for i, url in enumerate(page_urls): + link = ro_crate_link(signposting.find_signposting_http(url)) + crate_dir = tmp_path / f"{kind}-{i}" + crate_dir.mkdir() + metadata = crate_dir / "ro-crate-metadata.json" + metadata.write_text(requests.get(str(link.target), timeout=30).text) + report = crate_dir / "report.json" + + subprocess.run(["rocrate-validator", "validate", "--profile-identifier", "ro-crate-1.3", "--no-paging", + "--metadata-only", "--requirement-severity", "REQUIRED", "--output-format", "json", + "--output-file", str(report), str(metadata)], + stdin=subprocess.DEVNULL, capture_output=True, timeout=600) + + result = json.loads(report.read_text()) + issues = [f"{i['violatingEntity']}: {i['message']}" for i in result["issues"]] + assert result["passed"], f"{url} crate has issues:\n" + "\n".join(issues) diff --git a/test/unit/ro_crate_export_identifier_test.rb b/test/unit/ro_crate_export_identifier_test.rb new file mode 100644 index 000000000..57ab0c017 --- /dev/null +++ b/test/unit/ro_crate_export_identifier_test.rb @@ -0,0 +1,37 @@ +require 'test_helper' + +class RoCrateExportIdentifierTest < ActiveSupport::TestCase + test 'normalizes DOIs to https://doi.org/ URLs' do + expected = 'https://doi.org/10.1109/SEAA.2009.59' + ['10.1109/SEAA.2009.59', + 'doi:10.1109/SEAA.2009.59', + 'DOI:10.1109/SEAA.2009.59', + ' doi:10.1109/SEAA.2009.59 ', + 'http://doi.org/10.1109/SEAA.2009.59', + 'https://doi.org/10.1109/SEAA.2009.59', + 'http://dx.doi.org/10.1109/SEAA.2009.59', + 'https://dx.doi.org/10.1109/SEAA.2009.59', + 'https://www.doi.org/10.1109/SEAA.2009.59'].each do |doi| + assert_equal expected, RoCrateExport::Identifier.url(doi), doi + end + end + + test 'keeps other http(s) URLs' do + assert_equal 'http://dx.doi.ac.uk/2q3093032', RoCrateExport::Identifier.url('http://dx.doi.ac.uk/2q3093032') + assert_equal 'https://hdl.handle.net/1234/5678', RoCrateExport::Identifier.url('https://hdl.handle.net/1234/5678') + end + + test 'gives SPDX licences as SPDX license URIs' do + assert_equal 'http://spdx.org/licenses/CC-BY-4.0', RoCrateExport::Identifier.licence_url('CC-BY-4.0') + assert_equal 'http://spdx.org/licenses/Apache-2.0', RoCrateExport::Identifier.licence_url('Apache-2.0') + assert_nil RoCrateExport::Identifier.licence_url(nil) + assert_nil RoCrateExport::Identifier.licence_url('') + assert_nil RoCrateExport::Identifier.licence_url('not-a-licence') + end + + test 'rejects values that are not DOIs or URLs' do + [nil, '', 'not really a doi.', 'doi:', '10.12/too-short-prefix', 'https://doi.org/has spaces'].each do |value| + assert_nil RoCrateExport::Identifier.url(value), value.inspect + end + end +end