From f1d2e1ae6209a9b509375afd8a74e690608bacbe Mon Sep 17 00:00:00 2001 From: Finn Bacall Date: Mon, 20 Jul 2026 10:10:27 +0100 Subject: [PATCH 1/8] Add `origin_uri` column --- db/migrate/20260710154329_add_origin_uri_to_tables.rb | 6 ++++++ db/schema.rb | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20260710154329_add_origin_uri_to_tables.rb diff --git a/db/migrate/20260710154329_add_origin_uri_to_tables.rb b/db/migrate/20260710154329_add_origin_uri_to_tables.rb new file mode 100644 index 000000000..65b3d961b --- /dev/null +++ b/db/migrate/20260710154329_add_origin_uri_to_tables.rb @@ -0,0 +1,6 @@ +class AddOriginUriToTables < ActiveRecord::Migration[8.1] + def change + add_column :materials, :origin_uri, :string + add_column :events, :origin_uri, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index e87ff4c9b..5bec071c7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_04_21_144919) do +ActiveRecord::Schema[8.1].define(version: 2026_07_10_154329) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -212,6 +212,7 @@ t.text "notes" t.string "open_science", default: [], array: true t.string "organizer" + t.string "origin_uri" t.string "postcode" t.text "prerequisites" t.integer "presence", default: 0 @@ -371,6 +372,7 @@ t.date "last_scraped" t.text "learning_objectives" t.string "licence", default: "notspecified" + t.string "origin_uri" t.string "other_types" t.text "prerequisites" t.date "remote_created_date" From fa5aa3f166b8e28cb493c8ce249d2e3daf6067ec Mon Sep 17 00:00:00 2001 From: Finn Bacall Date: Mon, 20 Jul 2026 10:10:42 +0100 Subject: [PATCH 2/8] Remove redundant call --- lib/ingestors/event_ingestion.rb | 1 - lib/ingestors/material_ingestion.rb | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/ingestors/event_ingestion.rb b/lib/ingestors/event_ingestion.rb index 737a685e0..4dc1cca6c 100644 --- a/lib/ingestors/event_ingestion.rb +++ b/lib/ingestors/event_ingestion.rb @@ -6,7 +6,6 @@ def add_event(event) if event.is_a?(Hash) c = EventsController.new c.params = { event: event } - c.send(:event_params) event = OpenStruct.new(c.send(:event_params)) end event = handle_auto_parsing(event) diff --git a/lib/ingestors/material_ingestion.rb b/lib/ingestors/material_ingestion.rb index 81c8fadae..62c4ce407 100644 --- a/lib/ingestors/material_ingestion.rb +++ b/lib/ingestors/material_ingestion.rb @@ -6,7 +6,6 @@ def add_material(material) if material.is_a?(Hash) c = MaterialsController.new c.params = { material: material } - c.send(:material_params) material = OpenStruct.new(c.send(:material_params)) end material = handle_auto_parsing(material) From 252fd7a44229cea97d2621eaed5f824cb72f9000 Mon Sep 17 00:00:00 2001 From: Finn Bacall Date: Mon, 20 Jul 2026 10:11:20 +0100 Subject: [PATCH 3/8] Add resource type to OAI identifier --- app/models/application_record.rb | 3 +++ config/initializers/oai_provider.rb | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 7d2e97bf9..3594a98b2 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -6,4 +6,7 @@ class ApplicationRecord < ActiveRecord::Base include AutocompleteManager include ArrayFieldCleaner + def oai_identifier + "#{self.class.model_name.route_key}/#{id}" + end end diff --git a/config/initializers/oai_provider.rb b/config/initializers/oai_provider.rb index d2ae2a552..82a1594d4 100644 --- a/config/initializers/oai_provider.rb +++ b/config/initializers/oai_provider.rb @@ -12,12 +12,23 @@ def initialize end end +module OAI::Provider::Response + class RecordResponse < Base + private + + # Allow for a custom identifier to be used in the record without affecting the query (which happens with `identifier_field`) + def identifier_for(record) + "#{provider.prefix}:#{record.respond_to?(:oai_identifier) ? record.oai_identifier : record.send(provider.model.identifier_field)}" + end + end +end + class TrainingProvider < OAI::Provider::Base repository_name TeSS::Config.site['title'] repository_url "#{TeSS::Config.base_url}/oai-pmh" record_prefix "oai:#{URI(TeSS::Config.base_url).host}" admin_email TeSS::Config.contact_email - sample_id '142' # so that example id is oai:domain:142 + sample_id 'materials/142' # so that example id is oai:domain:materials/142 register_format(OAIRDF.instance) end From 539eb3058dbac42d2b7a7041b132569c51490799 Mon Sep 17 00:00:00 2001 From: Finn Bacall Date: Tue, 21 Jul 2026 12:58:43 +0100 Subject: [PATCH 4/8] Display origin info for resources with an `origin_uri` --- app/assets/stylesheets/application.scss | 5 +++++ app/helpers/application_helper.rb | 8 +++++++- app/views/common/_origin_info.erb | 12 ++++++++++++ app/views/events/_event.html.erb | 1 + app/views/events/show.html.erb | 1 + app/views/materials/_material.html.erb | 1 + app/views/materials/show.html.erb | 1 + config/locales/en.yml | 6 +++++- test/controllers/events_controller_test.rb | 15 +++++++++++++++ test/controllers/materials_controller_test.rb | 15 +++++++++++++++ 10 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 app/views/common/_origin_info.erb diff --git a/app/assets/stylesheets/application.scss b/app/assets/stylesheets/application.scss index f3974e377..9c8732585 100644 --- a/app/assets/stylesheets/application.scss +++ b/app/assets/stylesheets/application.scss @@ -690,6 +690,11 @@ sup { margin-left: 5px; } +.exchange-icon { + color: $brand-primary; + margin-left: 5px; +} + .days_ago_text { color: grey; font-style: italic; diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index dc9310434..524ec0593 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -29,7 +29,8 @@ module ApplicationHelper private: { icon: 'fa-eye-slash', message: 'This resource is private' }, missing: { icon: 'fa-chain-broken', message: 'This resource has been offline for over three days' }, check: { icon: 'fa-check', message: 'This resource is enabled' }, - cross: { icon: 'fa-times', message: 'This resource has been disabled' } + cross: { icon: 'fa-times', message: 'This resource has been disabled' }, + exchanged: { icon: 'fa-exchange', message: 'This resource originated from another TeSS registry' }, }.freeze # Countries that have priority in the country selection menu. Using ISO 3166-1 Alpha2 code. @@ -93,6 +94,11 @@ def event_status_icon(event, size = nil) end end + def exchange_icon(resource, size = nil) + return unless resource.origin_uri + "#{icon_for(:exchanged, size)}".html_safe + end + def icon_for(type, size = nil, options = {}) options[:class] ||= "info-icon#{'-' + size.to_s if size}" " +

<%= t('exchange.title') %>

+ +<% end %> diff --git a/app/views/events/_event.html.erb b/app/views/events/_event.html.erb index 687af7444..ce6c72eb1 100644 --- a/app/views/events/_event.html.erb +++ b/app/views/events/_event.html.erb @@ -19,6 +19,7 @@ <% end %> <%= event_status_icon(event) %> + <%= exchange_icon(event) %> <% if event.event_types.any? %> <% event.event_types.each do |t| %> diff --git a/app/views/events/show.html.erb b/app/views/events/show.html.erb index 3f036f328..c760c1e26 100644 --- a/app/views/events/show.html.erb +++ b/app/views/events/show.html.erb @@ -1,6 +1,7 @@
<%# SIDEBAR %>
<%= display_attribute_no_label(material, :resource_type) { |values| values.join(', ') } %> diff --git a/app/views/materials/show.html.erb b/app/views/materials/show.html.erb index 81131bcc6..0cd525900 100644 --- a/app/views/materials/show.html.erb +++ b/app/views/materials/show.html.erb @@ -3,6 +3,7 @@ <%= render partial: 'learning_paths/partials/navigation', locals: { topic_link: @learning_path_topic_link, topic_item: @learning_path_topic_item } if @learning_path_topic_link && @learning_path_topic_item %> + <%= render partial: 'common/origin_info', locals: { resource: @material } %> <%= render partial: 'content_providers/partials/content_provider_info', locals: { content_provider: @material.content_provider } %> <%= render partial: 'nodes/partials/associated_node_info', locals: { associated_nodes: @material.associated_nodes } %> <%= render(partial: 'users/partials/user_info', locals: { user: @material.user }) if current_user.try(:is_admin?) %> diff --git a/config/locales/en.yml b/config/locales/en.yml index 727d7ca70..5cb9f52b4 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1188,4 +1188,8 @@ en: any: Any time week: Within the last week month: Within the last month - year: Within the last year \ No newline at end of file + year: Within the last year + exchange: + title: Original entry + info: "This %{resource_type} originated from another TeSS registry. For complete, up-to-date information, please visit the original entry:" + link: Go to original entry \ No newline at end of file diff --git a/test/controllers/events_controller_test.rb b/test/controllers/events_controller_test.rb index e3004fe7b..984f6a0a6 100644 --- a/test/controllers/events_controller_test.rb +++ b/test/controllers/events_controller_test.rb @@ -1704,4 +1704,19 @@ class EventsControllerTest < ActionController::TestCase end end end + + test 'should link back to original entry if origin URI set' do + get :show, params: { id: @event } + assert_response :success + + assert_select '#origin-info', count: 0 + + uri = 'https://some-tess-instance.org/events/123' + @event.update(origin_uri: uri) + + get :show, params: { id: @event } + assert_response :success + + assert_select '#origin-info a[href=?]', uri + end end diff --git a/test/controllers/materials_controller_test.rb b/test/controllers/materials_controller_test.rb index 0d390b046..07f1d1b14 100644 --- a/test/controllers/materials_controller_test.rb +++ b/test/controllers/materials_controller_test.rb @@ -1706,4 +1706,19 @@ class MaterialsControllerTest < ActionController::TestCase assert_select 'p.scientific_topics a[href="http://edamontology.org/topic_0622"]', text: 'Genomics' assert_select 'p.operations a[href="http://edamontology.org/operation_0292"]', text: 'Sequence alignment' end + + test 'should link back to original entry if origin URI set' do + get :show, params: { id: @material } + assert_response :success + + assert_select '#origin-info', count: 0 + + uri = 'https://some-tess-instance.org/materials/123' + @material.update(origin_uri: uri) + + get :show, params: { id: @material } + assert_response :success + + assert_select '#origin-info a[href=?]', uri + end end From a615807cbacbc3d961a769b0194503ad133658ac Mon Sep 17 00:00:00 2001 From: Finn Bacall Date: Wed, 22 Jul 2026 11:29:14 +0100 Subject: [PATCH 5/8] Just mock requests rather than entire client. Rename file for consistency --- ...i_pmh_test.rb => oai_pmh_ingestor_test.rb} | 144 ++++++++++++------ 1 file changed, 97 insertions(+), 47 deletions(-) rename test/unit/ingestors/{oai_pmh_test.rb => oai_pmh_ingestor_test.rb} (66%) diff --git a/test/unit/ingestors/oai_pmh_test.rb b/test/unit/ingestors/oai_pmh_ingestor_test.rb similarity index 66% rename from test/unit/ingestors/oai_pmh_test.rb rename to test/unit/ingestors/oai_pmh_ingestor_test.rb index 3cb9acfb9..0dcbf5b0d 100644 --- a/test/unit/ingestors/oai_pmh_test.rb +++ b/test/unit/ingestors/oai_pmh_ingestor_test.rb @@ -1,47 +1,19 @@ require 'test_helper' -class FakeClient - def initialize(rdf_strings, dc_strings) - @rdf_response = Minitest::Mock.new - rdf_response = rdf_strings.map do |s| - inner_mock = Minitest::Mock.new - outer_mock = Minitest::Mock.new - inner_mock.expect(:metadata, outer_mock, []) - outer_mock.expect(:to_s, s, []) - inner_mock - end - dc_response = dc_strings.map do |s| - inner_mock = Minitest::Mock.new - outer_mock = Minitest::Mock.new - inner_mock.expect(:metadata, outer_mock, []) - outer_mock.expect(:to_s, s, []) - inner_mock - end - @rdf_response.expect(:full, rdf_response, []) - @dc_response = Minitest::Mock.new - @dc_response.expect(:full, dc_response, []) - end - - def list_records(metadata_prefix: nil) - if metadata_prefix == 'rdf' - @rdf_response - elsif metadata_prefix == 'oai_dc' - @dc_response - end - end -end - -class OaiPmhTest < ActiveSupport::TestCase +class OaiPmhIngestorTest < ActiveSupport::TestCase setup do @ingestor = Ingestors::OaiPmhIngestor.new @user = users(:regular_user) @content_provider = content_providers(:another_portal_provider) + @oai_pmh_url = 'https://example.org/oai-pmh' + @oai_pmh_id = 'example' end test 'should read empty oai pmh endpoint' do - OAI::Client.stub(:new, FakeClient.new([], [])) do - @ingestor.read('https://example.org') - end + mock_oai_pmh([], []) + + @ingestor.read(@oai_pmh_url) + assert_equal [], @ingestor.materials assert_equal [], @ingestor.events end @@ -68,9 +40,10 @@ class OaiPmhTest < ActiveSupport::TestCase METADATA - OAI::Client.stub(:new, FakeClient.new([], [record])) do - @ingestor.read('https://example.org') - end + mock_oai_pmh([], [record]) + + @ingestor.read(@oai_pmh_url) + result = @ingestor.materials.first assert_equal 'dc_title', result.title @@ -103,9 +76,10 @@ class OaiPmhTest < ActiveSupport::TestCase METADATA - OAI::Client.stub(:new, FakeClient.new([], [record])) do - @ingestor.read('https://example.org') - end + mock_oai_pmh([], [record]) + + @ingestor.read(@oai_pmh_url) + result = @ingestor.events.first assert_equal 'dc_title', result.title @@ -152,9 +126,9 @@ class OaiPmhTest < ActiveSupport::TestCase METADATA - OAI::Client.stub(:new, FakeClient.new([], [material1, material2, event1, event2])) do - @ingestor.read('https://example.org') - end + mock_oai_pmh([], [material1, material2, event1, event2]) + + @ingestor.read(@oai_pmh_url) assert_equal %w[title1 title2], @ingestor.events.map(&:title) assert_equal %w[title3 title4], @ingestor.materials.map(&:title) @@ -184,9 +158,9 @@ class OaiPmhTest < ActiveSupport::TestCase METADATA - OAI::Client.stub(:new, FakeClient.new([material, material, event], [])) do - @ingestor.read('https://example.org') - end + mock_oai_pmh([material, material, event], []) + + @ingestor.read(@oai_pmh_url) assert_equal 1, @ingestor.materials.length result = @ingestor.materials.first @@ -199,4 +173,80 @@ class OaiPmhTest < ActiveSupport::TestCase assert_equal 'bioschemas title2', result.title assert_equal 'https://example.org/bioschemas/event', result.url end + + private + + IDENTIFY = %( + + + 2026-07-22T09:37:47Z + https://tesshub.space/oai-pmh + + OAI-PMH Provider + https://some-oai-pmh.org/oai-pmh + 2.0 + contact@example.com + 2026-07-21T12:59:13Z + transient + YYYY-MM-DDThh:mm:ssZ + + + oai + some-oai-pmh + : + some-oai-pmh:142 + + + + +) + + LIST_RECORDS_WRAPPER = %( + + + + 2026-07-21T13:06:44Z + https://tesshub.space/oai-pmh + + %{records} + + +) + + + + # ?verb=Identify + # ?metadataPrefix=rdf&verb=ListRecords + # ?metadataPrefix=oai_dc&verb=ListRecords + def mock_oai_pmh(rdf_strings, dc_strings) + WebMock.stub_request(:get, "#{@oai_pmh_url}?verb=Identify").to_return(status: 200, body: IDENTIFY) + + id = 1 + rdf_response = LIST_RECORDS_WRAPPER % { records: rdf_strings.map do |rdf| + %( + +
+ oai:#{@oai_pmh_id}:#{id += 1} + #{Time.now.utc.iso8601} +
+ #{rdf} + + ) + end.join("\n") } + WebMock.stub_request(:get, "#{@oai_pmh_url}?metadataPrefix=rdf&verb=ListRecords").to_return(status: 200, body: rdf_response) + + dc_response = LIST_RECORDS_WRAPPER % { records: dc_strings.map do |rdf| + %( + +
+ oai:#{@oai_pmh_id}:#{id += 1} + #{Time.now.utc.iso8601} +
+ #{rdf} + + ) + end.join("\n") } + WebMock.stub_request(:get, "#{@oai_pmh_url}?metadataPrefix=oai_dc&verb=ListRecords").to_return(status: 200, body: dc_response) + end end From c59973c0f8d99ceca7b31c291a5f3d969d0b1b0c Mon Sep 17 00:00:00 2001 From: Finn Bacall Date: Wed, 22 Jul 2026 17:03:52 +0100 Subject: [PATCH 6/8] Identify TeSS instances via OAI-PMH, and store `origin_uri` for ingested resources --- app/controllers/events_controller.rb | 3 +- app/controllers/materials_controller.rb | 2 +- config/initializers/oai_provider.rb | 4 + lib/ingestors/oai_pmh_ingestor.rb | 18 +- test/unit/ingestors/oai_pmh_ingestor_test.rb | 11 ++ .../ingestors/tess_oai_pmh_identify.yml | 106 +++++++++++ .../ingestors/tess_oai_pmh_listrecords.yml | 166 ++++++++++++++++++ 7 files changed, 306 insertions(+), 4 deletions(-) create mode 100644 test/vcr_cassettes/ingestors/tess_oai_pmh_identify.yml create mode 100644 test/vcr_cassettes/ingestors/tess_oai_pmh_listrecords.yml diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb index 7df0b4b41..bdfe153fb 100644 --- a/app/controllers/events_controller.rb +++ b/app/controllers/events_controller.rb @@ -243,7 +243,8 @@ def event_params :timezone, :content_provider_id, { collection_ids: [] }, { node_ids: [] }, { node_names: [] }, { target_audience: [] }, { eligibility: [] }, :visible, { host_institutions: [] }, :capacity, :contact, :recognition, :learning_objectives, - :prerequisites, :tech_requirements, :cost_basis, :cost_value, :cost_currency, :language, :presence, + :prerequisites, :tech_requirements, :cost_basis, :cost_value, :cost_currency, :language, + :presence, :origin_uri, external_resources_attributes: %i[id url title _destroy], external_resources: %i[url title], material_ids: [], llm_interaction_attributes: %i[id scrape_or_process model prompt input output needs_processing _destroy], diff --git a/app/controllers/materials_controller.rb b/app/controllers/materials_controller.rb index c70360f89..9a53fa0d3 100644 --- a/app/controllers/materials_controller.rb +++ b/app/controllers/materials_controller.rb @@ -170,7 +170,7 @@ def material_params :last_scraped, :scraper_record, :remote_created_date, :remote_updated_date, :content_provider_id, :difficulty_level, :version, :status, :date_created, :date_modified, :date_published, :other_types, - :prerequisites, :syllabus, :visible, :learning_objectives, { subsets: [] }, + :prerequisites, :syllabus, :visible, :learning_objectives, :origin_uri, { subsets: [] }, { target_audience: [] }, { collection_ids: [] }, { keywords: [] }, { resource_type: [] }, { scientific_topic_names: [] }, { scientific_topic_uris: [] }, diff --git a/config/initializers/oai_provider.rb b/config/initializers/oai_provider.rb index 82a1594d4..b120a629c 100644 --- a/config/initializers/oai_provider.rb +++ b/config/initializers/oai_provider.rb @@ -29,6 +29,10 @@ class TrainingProvider < OAI::Provider::Base record_prefix "oai:#{URI(TeSS::Config.base_url).host}" admin_email TeSS::Config.contact_email sample_id 'materials/142' # so that example id is oai:domain:materials/142 + extra_description %( + + + ) register_format(OAIRDF.instance) end diff --git a/lib/ingestors/oai_pmh_ingestor.rb b/lib/ingestors/oai_pmh_ingestor.rb index fff2426e1..7ffa6b2ff 100644 --- a/lib/ingestors/oai_pmh_ingestor.rb +++ b/lib/ingestors/oai_pmh_ingestor.rb @@ -22,8 +22,11 @@ def initialize def read(source_url) client = OAI::Client.new source_url, headers: { 'From' => config[:mail], 'User-Agent' => config[:user_agent] } + + tess_instance = client.identify.descriptions.any? { |d| d.get_elements('//tess-instance').any? } + found_bioschemas = begin - read_oai_rdf(client) + read_oai_rdf(client, tess_instance: tess_instance) rescue OAI::ArgumentException false end @@ -75,7 +78,7 @@ def read_oai_dublin_core(client) @messages << "found #{count} records" end - def read_oai_rdf(client) + def read_oai_rdf(client, tess_instance: false) provider_events = [] provider_materials = [] totals = Hash.new(0) @@ -86,6 +89,17 @@ def read_oai_rdf(client) output = parse_bioschemas(bioschemas_xml) next unless output + extra_metadata = {} + if tess_instance + origin_uri = metadata_tag.at_xpath('//rdf:RDF/*/@rdf:about', 'rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#')&.to_s + extra_metadata[:origin_uri] = origin_uri + end + + extra_metadata.each do |key, value| + output[:resources][:events].each { |r| r[key] ||= value } + output[:resources][:materials].each { |r| r[key] ||= value } + end + provider_events += output[:resources][:events] provider_materials += output[:resources][:materials] output[:totals].each do |key, value| diff --git a/test/unit/ingestors/oai_pmh_ingestor_test.rb b/test/unit/ingestors/oai_pmh_ingestor_test.rb index 0dcbf5b0d..919b8ea82 100644 --- a/test/unit/ingestors/oai_pmh_ingestor_test.rb +++ b/test/unit/ingestors/oai_pmh_ingestor_test.rb @@ -174,6 +174,17 @@ class OaiPmhIngestorTest < ActiveSupport::TestCase assert_equal 'https://example.org/bioschemas/event', result.url end + test 'should read TeSS instance OAI-PMH endpoint and store origin URI' do + VCR.use_cassette('ingestors/tess_oai_pmh_listrecords') do + VCR.use_cassette('ingestors/tess_oai_pmh_identify') do + @ingestor.read('https://oai-pmh.tesshub.space/oai-pmh') + end + end + + assert_equal 2, @ingestor.materials.length + assert @ingestor.materials.all? { |m| m.origin_uri.start_with?('https://tesshub.space/materials/') } + end + private IDENTIFY = %( diff --git a/test/vcr_cassettes/ingestors/tess_oai_pmh_identify.yml b/test/vcr_cassettes/ingestors/tess_oai_pmh_identify.yml new file mode 100644 index 000000000..aa0e4bd6b --- /dev/null +++ b/test/vcr_cassettes/ingestors/tess_oai_pmh_identify.yml @@ -0,0 +1,106 @@ +--- +http_interactions: +- request: + method: get + uri: https://oai-pmh.tesshub.space/oai-pmh?verb=Identify + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0 + From: + - test@example.com + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 22 Jul 2026 09:37:47 GMT + Content-Type: + - text/xml; charset=utf-8 + Content-Length: + - '1229' + Connection: + - keep-alive + Server: + - cloudflare + Cache-Control: + - max-age=0, private, must-revalidate + Vary: + - Accept,Origin,Accept-Encoding + Strict-Transport-Security: + - max-age=63072000; includeSubDomains + Referrer-Policy: + - strict-origin-when-cross-origin + X-Permitted-Cross-Domain-Policies: + - none + X-Xss-Protection: + - '0' + Feature-Policy: + - camera 'none'; gyroscope 'none'; microphone 'none'; usb 'none'; fullscreen + 'self'; payment 'none' + X-Request-Id: + - d0398130-56e7-4e4a-afb9-926d2f97a4a7 + X-Frame-Options: + - SAMEORIGIN + X-Runtime: + - '0.016690' + X-Content-Type-Options: + - nosniff + Content-Security-Policy: + - 'default-src ''self'' https:; font-src ''self'' https: data:; img-src ''self'' + https: data:; object-src ''none''; script-src ''self'' https: ''unsafe-inline''; + style-src ''self'' https: ''unsafe-inline''' + X-Powered-By: + - Phusion Passenger(R) 6.0.18 + Etag: + - W/"0bcac93baeefbfbe7c7335e2fb044adf-gzip" + Status: + - 200 OK + Nel: + - '{"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}' + Report-To: + - '{"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=tgOwpWlqq7CGsthz3Gbpkvrxzfa63OAWYsItZtDASLiXdjA16f%2B2b%2B1Mv9vfjbyA4l0c6xk6BnljhdR1oSOJOAFhudUujuVZ%2B8v53ANz0eqhB%2BsYt5QIQtuN%2F9eMJx3df4uDL6mChQU%3D"}]}' + Cf-Cache-Status: + - DYNAMIC + Cf-Ray: + - a1f177feba4f467f-LHR + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: |- + + + + 2026-07-22T09:37:47Z + https://tesshub.space/oai-pmh + + TeSSHub + https://tesshub.space/oai-pmh + 2.0 + contact@example.com + 2026-07-21T12:59:13Z + transient + YYYY-MM-DDThh:mm:ssZ + + + oai + tesshub.space + : + oai:tesshub.space:142 + + + + + + + + recorded_at: Wed, 22 Jul 2026 09:37:47 GMT +recorded_with: VCR 6.2.0 diff --git a/test/vcr_cassettes/ingestors/tess_oai_pmh_listrecords.yml b/test/vcr_cassettes/ingestors/tess_oai_pmh_listrecords.yml new file mode 100644 index 000000000..65d07fbf9 --- /dev/null +++ b/test/vcr_cassettes/ingestors/tess_oai_pmh_listrecords.yml @@ -0,0 +1,166 @@ +--- +http_interactions: +- request: + method: get + uri: https://oai-pmh.tesshub.space/oai-pmh?metadataPrefix=rdf&verb=ListRecords + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0 + From: + - test@example.com + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 21 Jul 2026 13:06:44 GMT + Content-Type: + - text/xml; charset=utf-8 + Content-Length: + - '3270' + Connection: + - keep-alive + Server: + - cloudflare + Cache-Control: + - max-age=0, private, must-revalidate + Vary: + - Accept,Origin,Accept-Encoding + Strict-Transport-Security: + - max-age=63072000; includeSubDomains + Referrer-Policy: + - strict-origin-when-cross-origin + X-Permitted-Cross-Domain-Policies: + - none + X-Xss-Protection: + - '0' + Feature-Policy: + - camera 'none'; gyroscope 'none'; microphone 'none'; usb 'none'; fullscreen + 'self'; payment 'none' + X-Request-Id: + - d0013d21-bc24-4956-b930-055f3f7c5956 + X-Frame-Options: + - SAMEORIGIN + X-Runtime: + - '0.071256' + X-Content-Type-Options: + - nosniff + Content-Security-Policy: + - 'default-src ''self'' https:; font-src ''self'' https: data:; img-src ''self'' + https: data:; object-src ''none''; script-src ''self'' https: ''unsafe-inline''; + style-src ''self'' https: ''unsafe-inline''' + X-Powered-By: + - Phusion Passenger(R) 6.0.18 + Etag: + - W/"a3f07edfcb2ab6214614661382a7b916-gzip" + Status: + - 200 OK + Nel: + - '{"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}' + Report-To: + - '{"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=%2B%2FNroeqG4tsUqXtpDqDYWQwXVwloSkQ7imaOxw87RJRDIJsuNsAue9n%2Bx1hmc7Sebtdnt69rRi8vFg6Vv3L7NcHaWXd0IWe8r4ymgf744QWJwtIxvIbE%2B3vRftGM%2F2IokT%2BF9wYNAFE%3D"}]}' + Cf-Cache-Status: + - DYNAMIC + Cf-Ray: + - a1ea6cb55b4525c5-LHR + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: |- + + + + 2026-07-21T13:06:44Z + https://tesshub.space/oai-pmh + + +
+ oai:tesshub.space:3931 + 2026-07-21T12:59:13Z +
+ + + + + + + + Example Material + + An example material + example + + + Steve Steveson + + + + + Josiah Carberry + https://orcid.org/0000-0002-1825-0097 + + + + + Joe Bloggs + + + + + + + + +
+ oai:tesshub.space:3932 + 2026-07-21T13:04:24Z +
+ + + + + + + + Another Material + + Sir, a second material has hit the registry + testing + example + + + Jose Gonzales + + + + + Amelie Martin + + + + + Researchers + + + + + + + + + + recorded_at: Tue, 21 Jul 2026 13:06:44 GMT +recorded_with: VCR 6.2.0 From 4deed0b4370bf6cb39de295acfa4127dde84783a Mon Sep 17 00:00:00 2001 From: Finn Bacall Date: Thu, 23 Jul 2026 10:08:51 +0100 Subject: [PATCH 7/8] Validate origin URI. Address review comments --- app/helpers/application_helper.rb | 2 +- app/models/event.rb | 2 ++ app/models/material.rb | 1 + lib/ingestors/oai_pmh_ingestor.rb | 2 +- test/models/material_test.rb | 17 +++++++++++++++++ 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 524ec0593..e5f9ca7c1 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -95,7 +95,7 @@ def event_status_icon(event, size = nil) end def exchange_icon(resource, size = nil) - return unless resource.origin_uri + return unless resource.origin_uri.present? "#{icon_for(:exchanged, size)}".html_safe end diff --git a/app/models/event.rb b/app/models/event.rb index 571acf006..5e62510d6 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -157,6 +157,8 @@ class Event < ApplicationRecord validates :presence, inclusion: { in: presences.keys, allow_blank: true } validates :keywords, length: { maximum: 20 } validate :allowed_url + validates :origin_uri, url: { allow_blank: true } + clean_array_fields(:keywords, :fields, :event_types, :target_audience, :eligibility, :host_institutions, :sponsors) update_suggestions(:keywords, :target_audience, :host_institutions) diff --git a/app/models/material.rb b/app/models/material.rb index f4ac0cb11..934edff78 100644 --- a/app/models/material.rb +++ b/app/models/material.rb @@ -123,6 +123,7 @@ class Material < ApplicationRecord validates :url, url: true validates :other_types, presence: true, if: proc { |m| m.resource_type.include?('other') } validates :keywords, length: { maximum: 20 } + validates :origin_uri, url: { allow_blank: true } clean_array_fields(:keywords, :fields, :target_audience, :resource_type, :subsets) diff --git a/lib/ingestors/oai_pmh_ingestor.rb b/lib/ingestors/oai_pmh_ingestor.rb index 7ffa6b2ff..386eb1a48 100644 --- a/lib/ingestors/oai_pmh_ingestor.rb +++ b/lib/ingestors/oai_pmh_ingestor.rb @@ -91,7 +91,7 @@ def read_oai_rdf(client, tess_instance: false) extra_metadata = {} if tess_instance - origin_uri = metadata_tag.at_xpath('//rdf:RDF/*/@rdf:about', 'rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#')&.to_s + origin_uri = metadata_tag.at_xpath('//rdf:RDF/*/@rdf:about', 'rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#')&.value extra_metadata[:origin_uri] = origin_uri end diff --git a/test/models/material_test.rb b/test/models/material_test.rb index c2d98aa5c..f5e35e861 100644 --- a/test/models/material_test.rb +++ b/test/models/material_test.rb @@ -751,4 +751,21 @@ class MaterialTest < ActiveSupport::TestCase bioschemas = m.to_bioschemas.first.generate refute bioschemas.key?(:license) end + + test 'validates origin_uri' do + @material.origin_uri = nil + assert @material.valid? + + @material.origin_uri = '' + assert @material.valid? + + @material.origin_uri = 'https://tess-instance.org/materials/123' + assert @material.valid? + + @material.origin_uri = 'hamster' + refute @material.valid? + + @material.origin_uri = 'ftp://tess-instance.org/materials/123' + refute @material.valid? + end end From 40f5cf4ac177ab6b749ecb6e47d7b11313da3fbc Mon Sep 17 00:00:00 2001 From: Finn Bacall Date: Fri, 24 Jul 2026 18:12:15 +0100 Subject: [PATCH 8/8] Use `tesshub.org` as xmlns Co-authored-by: Martin Voigt --- config/initializers/oai_provider.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/oai_provider.rb b/config/initializers/oai_provider.rb index b120a629c..97e58b827 100644 --- a/config/initializers/oai_provider.rb +++ b/config/initializers/oai_provider.rb @@ -31,7 +31,7 @@ class TrainingProvider < OAI::Provider::Base sample_id 'materials/142' # so that example id is oai:domain:materials/142 extra_description %( - + ) register_format(OAIRDF.instance)