diff --git a/argus/VERSION b/argus/VERSION index 75cd66fb9..53270b0cd 100644 --- a/argus/VERSION +++ b/argus/VERSION @@ -1,7 +1,7 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VER=2.0-BETA-07 +VER=2.0-BETA-08 TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")" unset VER diff --git a/argus/build.gradle b/argus/build.gradle index d12b2de67..fe9861f14 100644 --- a/argus/build.gradle +++ b/argus/build.gradle @@ -51,6 +51,7 @@ dependencies { runtimeOnly 'org.opencadc:cadc-log:[1.1.0,2.0)' // force min versions for CVEs + runtimeOnly 'org.apache.commons:commons-configuration2:[2.15.0,)' runtimeOnly 'org.apache.commons:commons-lang3:[3.18.0,)' runtimeOnly 'com.nimbusds:nimbus-jose-jwt:[9.37.4,)' // this breaks CI build on gradle 6 because of library variants @@ -92,5 +93,6 @@ configurations { runtimeOnly.exclude group: 'org.eclipse.jetty' runtimeOnly.exclude group: 'org.eclipse.jetty' runtimeOnly.exclude group: 'io.netty' + runtimeOnly.exclude group: 'org.jline' } diff --git a/argus/src/main/java/org/opencadc/argus/tap/InitCaomTapSchemaContent.java b/argus/src/main/java/org/opencadc/argus/tap/InitCaomTapSchemaContent.java index 45b5125fb..4d7d34945 100644 --- a/argus/src/main/java/org/opencadc/argus/tap/InitCaomTapSchemaContent.java +++ b/argus/src/main/java/org/opencadc/argus/tap/InitCaomTapSchemaContent.java @@ -87,7 +87,7 @@ public class InitCaomTapSchemaContent extends InitDatabase { private static final Logger log = Logger.getLogger(InitCaomTapSchemaContent.class); public static final String MODEL_NAME = "caom2-schema"; - public static final String MODEL_VERSION = "2.5.0-P"; + public static final String MODEL_VERSION = "2.5.0-beta-2"; // the SQL is tightly coupled to cadc-tap-schema table names (for TAP-1.1) static String[] BASE_SQL = new String[] { @@ -114,7 +114,9 @@ public class InitCaomTapSchemaContent extends InitDatabase { public InitCaomTapSchemaContent(DataSource dataSource, String database, String schema, boolean extras) { // use MODELVERSION/extras so changing extras will cause a recreate // eg 1.2.13/false <-> 1.2.13/true - super(dataSource, database, schema, MODEL_NAME, MODEL_VERSION + "/" + extras); + //super(dataSource, database, schema, MODEL_NAME, MODEL_VERSION + "/" + extras); + // TODO: currently no extras in 2.5 beta + super(dataSource, database, schema, MODEL_NAME, MODEL_VERSION); String[] src = BASE_SQL; if (extras) { src = BASE_EXTRA_SQL; diff --git a/argus/src/main/resources/sql/caom2.tap_schema_content11.sql b/argus/src/main/resources/sql/caom2.tap_schema_content11.sql index 91ba2c52b..0efa187bf 100644 --- a/argus/src/main/resources/sql/caom2.tap_schema_content11.sql +++ b/argus/src/main/resources/sql/caom2.tap_schema_content11.sql @@ -81,7 +81,7 @@ insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,u ( 'caom2.Observation', 'targetPosition_equinox', 'equinox of target position coordinate system', 'caom2:TargetPosition.equinox', NULL, NULL, 'double', NULL,NULL, 0,0,0 , 41), ( 'caom2.Observation', 'targetPosition_coordinates', 'intended position to observe', 'caom2:TargetPosition.coordinates', NULL, 'deg', 'double', '2', 'point', 0,0,0 , 41), -( 'caom2.Observation', 'telescope_name', 'name of telescope used to acquire observation', 'caom2:Telescope.name', NULL, NULL, 'char', '32*',NULL, 1,0,0 , 45), +( 'caom2.Observation', 'telescope_name', 'name of telescope used to acquire observation', 'caom2:Telescope.name', NULL, NULL, 'char', '32*',NULL, 1,1,1 , 45), ( 'caom2.Observation', 'telescope_geoLocationX', 'x component of geocentric location of telescope', 'caom2:Telescope.geoLocationX', NULL, NULL, 'double', NULL, NULL, 0,0,0 , 46), ( 'caom2.Observation', 'telescope_geoLocationY', 'y component of geocentric location of telescope', 'caom2:Telescope.geoLocationY', NULL, NULL, 'double', NULL, NULL, 0,0,0 , 47), ( 'caom2.Observation', 'telescope_geoLocationZ', 'z component of geocentric location of telescope', 'caom2:Telescope.geoLocationZ', NULL, NULL, 'double', NULL, NULL, 0,0,0 , 48), @@ -90,7 +90,7 @@ insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,u ( 'caom2.Observation', 'requirements_flag', 'flag describing satisfied proposal requirements (possible values: fail)', 'caom2:Observation.requirements.flag', NULL, NULL, 'char', '16',NULL, 1,0,0 , 50), -( 'caom2.Observation', 'instrument_name', 'name of instrument used to acquire observation', 'caom2:Instrument.name', NULL, NULL, 'char', '32*',NULL, 1,0,0 , 60), +( 'caom2.Observation', 'instrument_name', 'name of instrument used to acquire observation', 'caom2:Instrument.name', NULL, NULL, 'char', '32*',NULL, 1,1,1 , 60), ( 'caom2.Observation', 'instrument_keywords', 'instrument keywords (separated by |)', 'caom2:Instrument.keywords', NULL, NULL, 'char', '*',NULL, 0,0,0 , 61), ( 'caom2.Observation', 'environment_seeing', 'atmospheric seeing (FWHM)', 'caom2:Environment.seeing', NULL, 'arcsec', 'double', NULL, NULL, 0,0,0 , 70), @@ -104,12 +104,12 @@ insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,u ( 'caom2.Observation', 'members', 'members of a composite observation (space-separated list of Observation URIs)', 'caom2:Observation.members', NULL, NULL, 'char','*',NULL, 0,0,0 , 80), ( 'caom2.Observation', 'typeCode', 'single character code to denote type: S(impleObservation) or D(erivedOvservation)', 'caom2:Observation.typeCode', NULL, NULL, 'char',NULL,NULL, 0,0,0 , 81), -( 'caom2.Observation', 'obsID', 'primary key', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 0,1,0 , 120), -( 'caom2.Observation', 'metaProducer', 'identifier for the producer of this entity metadata (URI of the form {organisation}:{software}-{version}) [new in 2.4]', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 121), -( 'caom2.Observation', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*','timestamp', 1,0,0 , 122), -( 'caom2.Observation', 'maxLastModified', 'timestamp of last modification of this entity+children', 'caom2:CaomEntity.maxLastModified', NULL, NULL, 'char', '23*','timestamp', 1,0,0 , 123), -( 'caom2.Observation', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 124), -( 'caom2.Observation', 'accMetaChecksum', 'checksum of the metadata in this entity+children (URI of the form {algorithm}:{hex value})', 'caom2:CaomEntity.accMetaChecksum', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 125) +( 'caom2.Observation', 'obsID', 'primary key', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 1,1,1 , 120), +( 'caom2.Observation', 'metaProducer', 'identifier for the producer of this entity metadata (URI of the form {organisation}:{software}-{version}) [new in 2.4]', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 121), +( 'caom2.Observation', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*','timestamp', 0,0,0 , 122), +( 'caom2.Observation', 'maxLastModified', 'timestamp of last modification of this entity+children', 'caom2:CaomEntity.maxLastModified', NULL, NULL, 'char', '23*','timestamp', 0,1,0 , 123), +( 'caom2.Observation', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 124), +( 'caom2.Observation', 'accMetaChecksum', 'checksum of the metadata in this entity+children (URI of the form {algorithm}:{hex value})', 'caom2:CaomEntity.accMetaChecksum', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 125) ; -- Plane @@ -147,7 +147,7 @@ insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,u ( 'caom2.Plane', 'observable_calibration', 'calibration status of the observed/measure quantity [new in 2.5]', 'caom2:Observable.calibration', NULL, NULL, 'char', '64*', NULL, 0,0,0 , 41), ( 'caom2.Plane', 'quality_flag', 'flag describing the data quality (possible values: junk)', 'caom2:Quality.flag', NULL, NULL, 'char', '16*', NULL, 0,0,0 , 42), -( 'caom2.Plane', 'position_bounds', 'queryable positional coverage of the data', 'caom2:Position.bounds', NULL, 'deg', 'char', '*', 'shape', 0,0,0 , 50), +( 'caom2.Plane', 'position_bounds', 'queryable positional coverage of the data', 'caom2:Position.bounds', NULL, 'deg', 'char', '*', 'shape', 1,1,1 , 50), ( 'caom2.Plane', 'position_samples', 'detailed positional coverage of the data', 'caom2:Position.samples', NULL, 'deg', 'char', '*', 'multishape', 0,0,0 , 51), ( 'caom2.Plane', 'position_minBounds', 'queryable minimum positional coverage... TBD [new in 2.5]', 'caom2:Position.minBounds', NULL, 'deg', 'char', '*', 'shape', 0,0,0 , 52), ( 'caom2.Plane', 'position_dimension', 'dimensions (number of pixels) along the spatial axes', 'caom2:Position.dimension', NULL, NULL, 'long', '2', NULL, 0,0,0 , 53), @@ -157,7 +157,7 @@ insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,u ( 'caom2.Plane', 'position_sampleSize', 'median sample (pixel) size on spatial axes', 'caom2:Position.sampleSize', NULL, 'arcsec', 'double', NULL, NULL, 0,0,0 , 56), ( 'caom2.Plane', 'position_calibration', 'calibration of the position metadata (astrometry) [new in 2.5]', 'caom2:Position.timeDependent', NULL, NULL, 'char', '64*', NULL, 0,0,0 , 56), -( 'caom2.Plane', 'energy_bounds', 'queryable energy coverage', 'caom2:Energy.bounds', NULL, 'm', 'double', '2','interval', 0,1,0 , 61), +( 'caom2.Plane', 'energy_bounds', 'queryable energy coverage', 'caom2:Energy.bounds', NULL, 'm', 'double', '2','interval', 1,1,1 , 61), ( 'caom2.Plane', 'energy_samples', 'detailed energy coverage', 'caom2:Energy.samples', NULL, 'm', 'double', '*', 'multiinterval', 0,0,0 , 62), ( 'caom2.Plane', 'energy_dimension', 'dimension (number of pixels) along energy axis', 'caom2:Energy.dimension', NULL, NULL, 'long', NULL, NULL, 0,0,0 , 63), ( 'caom2.Plane', 'energy_resolvingPower', 'representative relative energy resolution aka resolving power (R)', 'caom2:Energy.resolvingPower', NULL, NULL, 'double', NULL, NULL, 0,0,0 , 66), @@ -172,17 +172,17 @@ insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,u ( 'caom2.Plane', 'energy_transition_species', 'atom or molecule', 'caom2:Transition.species', NULL, NULL, 'char', '32*',NULL, 0,0,0 , 73), ( 'caom2.Plane', 'energy_transition_transition', 'specific energy transition of species', 'caom2:Transition.transition', NULL, NULL, 'char', '32*', NULL, 0,0,0 , 74), -( 'caom2.Plane', 'time_bounds', 'time coverage (Modified Julian Day)', 'caom2:Time.bounds', NULL, 'd', 'double', '2', 'interval', 0,1,0 , 80), +( 'caom2.Plane', 'time_bounds', 'time coverage (Modified Julian Day)', 'caom2:Time.bounds', NULL, 'd', 'double', '2', 'interval', 1,1,1 , 80), ( 'caom2.Plane', 'time_samples', 'detailed time coverage (Modified Julian Day)', 'caom2:Time.bounds.samples', NULL, 'd', 'double', '*', 'multiinterval', 0,0,0 , 81), ( 'caom2.Plane', 'time_dimension', 'dimension (number of pixels) along time axis', 'caom2:Time.dimension', NULL, NULL, 'long', NULL, NULL, 0,0,0 , 84), ( 'caom2.Plane', 'time_resolution', 'median resolution on the time axis', 'caom2:Time.resolution', NULL, 'd', 'double', NULL, NULL, 0,0,0 , 85), ( 'caom2.Plane', 'time_resolutionBounds', 'range of resolution on the time axis [new in 2.4]', 'caom2:Time.resolutionBounds', NULL, 'd', 'double', '2', 'interval', 0,0,0 , 86), -( 'caom2.Plane', 'time_exposure', 'representative exposure time (mean per pixel?)', 'caom2:Time.exposure', NULL, 's', 'double', NULL, NULL, 0,0,0 , 88), +( 'caom2.Plane', 'time_exposure', 'representative exposure time (mean per pixel?)', 'caom2:Time.exposure', NULL, 's', 'double', NULL, NULL, 1,1,1 , 88), ( 'caom2.Plane', 'time_exposureBounds', 'range of exposure on the time axis [new in 2.4]', 'caom2:Time.exposureBounds', NULL, 's', 'double', '2', 'interval', 0,0,0 , 86), ( 'caom2.Plane', 'time_sampleSize', 'median sample (pixel) size on time axis', 'caom2:Time.sampleSize', NULL, 'd', 'double', NULL, NULL, 0,0,0 , 87), ( 'caom2.Plane', 'time_calibration', 'calibration of the time axis', 'caom2:Time.calibration', NULL, NULL, 'char', '64*', NULL, 0,0,0 , 88), -( 'caom2.Plane', 'polarization_states', 'polarization letter codes in canonical order, separated by /', 'caom2:Polarization.states', NULL, NULL, 'char', '32*', NULL, 0,0,0 , 90), +( 'caom2.Plane', 'polarization_states', 'polarization letter codes in canonical order, separated by /', 'caom2:Polarization.states', NULL, NULL, 'char', '32*', NULL, 1,1,1 , 90), ( 'caom2.Plane', 'polarization_dimension', 'number of samples (pixels) along polarization axis', 'caom2:Polarization.dimension', NULL, NULL, 'int', NULL, NULL, 0,0,0 , 91), ( 'caom2.Plane', 'custom_ctype', 'coordinate type for custom axis', 'caom2:CustomAxis.ctype', NULL, NULL, 'char', '32*', NULL, 0,0,0 , 100), @@ -194,18 +194,18 @@ insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,u ( 'caom2.Plane', 'uv_distributionEccentricity', 'TBD [new in 2.5]', 'caom2:Visibility.distributionEccentricity', NULL, NULL, 'double', NULL, NULL, 0,0,0 , 111), ( 'caom2.Plane', 'uv_distributionFill', 'TBD [new in 2.5]', 'caom2:Visibility.distributionFill', NULL, NULL, 'double', NULL, NULL, 0,0,0 , 112), -( 'caom2.Plane', 'planeID', 'unique plane identifier', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 0,1,0 , 120), -( 'caom2.Plane', 'metaProducer', 'identifier for the producer of this entity metadata (URI of the form {organisation}:{software}-{version}) [new in 2.4]', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 121), -( 'caom2.Plane', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*','timestamp', 1,1,0 , 122), -( 'caom2.Plane', 'maxLastModified', 'timestamp of last modification of this entity+children', 'caom2:CaomEntity.maxLastModified', NULL, NULL, 'char', '23*','timestamp', 1,1,0 , 123), -( 'caom2.Plane', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 124), -( 'caom2.Plane', 'accMetaChecksum', 'checksum of the metadata in this entity+children (URI of the form {algorithm}:{hex value})', 'caom2:CaomEntity.accMetaChecksum', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 125) +( 'caom2.Plane', 'planeID', 'unique plane identifier', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 1,1,1 , 120), +( 'caom2.Plane', 'metaProducer', 'identifier for the producer of this entity metadata (URI of the form {organisation}:{software}-{version}) [new in 2.4]', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 121), +( 'caom2.Plane', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*','timestamp', 0,0,0 , 122), +( 'caom2.Plane', 'maxLastModified', 'timestamp of last modification of this entity+children', 'caom2:CaomEntity.maxLastModified', NULL, NULL, 'char', '23*','timestamp', 0,0,0 , 123), +( 'caom2.Plane', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 124), +( 'caom2.Plane', 'accMetaChecksum', 'checksum of the metadata in this entity+children (URI of the form {algorithm}:{hex value})', 'caom2:CaomEntity.accMetaChecksum', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 125) ; -- Artifact insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,unit,datatype,arraysize,xtype, principal,indexed,std,column_index,column_id) values -( 'caom2.Artifact', 'uri', 'external URI for the physical artifact', 'caom2:Artifact.uri', NULL, NULL, 'char', '*','uri', 1,1,0 , 1, 'caomArtifactURI') +( 'caom2.Artifact', 'uri', 'external URI for the physical artifact', 'caom2:Artifact.uri', NULL, NULL, 'char', '*','uri', 1,1,1 , 1, 'caomArtifactURI') ; insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,unit,datatype,arraysize,xtype,principal,indexed,std,column_index) values ( 'caom2.Artifact', 'uriBucket', 'short hex string', 'caom2:Artifact.uriBucket', NULL, NULL, 'char','3',NULL, 0,1,0 , 2), @@ -220,37 +220,37 @@ insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,u ( 'caom2.Artifact', 'contentReadGroups', 'GMS groups that are authorized to retrieve the artifact (default: inherit from Plane) [new in 2.4]', 'caom2:Artifact.contentReadGroups', NULL, NULL, 'char', '*',NULL, 0,0,0, 21), ( 'caom2.Artifact', 'descriptionID', 'reference to an ArtifactDescription [new in 2.5]', 'caom2:Artifact.descriptionID', NULL, NULL, 'char', '*','uri', 0,0,0, 22), -( 'caom2.Artifact', 'artifactID', 'primary key', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 0,1,0 , 40), -( 'caom2.Artifact', 'metaProducer', 'identifier for the producer of this entity metadata (URI of the form {organisation}:{software}-{version}) [new in 2.4]', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 41), -( 'caom2.Artifact', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*', 'timestamp', 1,1,0, 42), -( 'caom2.Artifact', 'maxLastModified', 'timestamp of last modification of this entity+children', 'caom2:CaomEntity.maxLastModified', NULL, NULL, 'char', '23*', 'timestamp', 1,1,0, 43), -( 'caom2.Artifact', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 44), -( 'caom2.Artifact', 'accMetaChecksum', 'checksum of the metadata in this entity+children (URI of the form {algorithm}:{hex value})', 'caom2:CaomEntity.accMetaChecksum', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 45) +( 'caom2.Artifact', 'artifactID', 'primary key', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 1,1,1 , 40), +( 'caom2.Artifact', 'metaProducer', 'identifier for the producer of this entity metadata (URI of the form {organisation}:{software}-{version}) [new in 2.4]', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 41), +( 'caom2.Artifact', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*', 'timestamp', 0,0,0, 42), +( 'caom2.Artifact', 'maxLastModified', 'timestamp of last modification of this entity+children', 'caom2:CaomEntity.maxLastModified', NULL, NULL, 'char', '23*', 'timestamp', 0,0,0, 43), +( 'caom2.Artifact', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 44), +( 'caom2.Artifact', 'accMetaChecksum', 'checksum of the metadata in this entity+children (URI of the form {algorithm}:{hex value})', 'caom2:CaomEntity.accMetaChecksum', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 45) ; insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,unit,datatype,arraysize,xtype,principal,indexed,std,column_index) values -( 'caom2.ArtifactDescription', 'uri', 'logical identifier', 'caom2:ArtifactDescription.uri', NULL, NULL, 'char','*','uri', 0,1,0 , 1), -( 'caom2.ArtifactDescription', 'description', 'human-readable description of a class of artifacts', 'caom2:ArtifactDescription.description', NULL, NULL, 'char','*',NULL, 0,1,0 , 2), -( 'caom2.ArtifactDescription', 'id', 'primary key', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 0,1,0 , 3), -( 'caom2.ArtifactDescription', 'metaProducer', 'identifier for the producer of this entity metadata (URI of the form {organisation}:{software}-{version}) [new in 2.4]', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 4), -( 'caom2.ArtifactDescription', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*', 'timestamp', 1,1,0, 5), -( 'caom2.ArtifactDescription', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 6) +( 'caom2.ArtifactDescription', 'uri', 'logical identifier', 'caom2:ArtifactDescription.uri', NULL, NULL, 'char','*','uri', 1,1,1 , 1), +( 'caom2.ArtifactDescription', 'description', 'human-readable description of a class of artifacts', 'caom2:ArtifactDescription.description', NULL, NULL, 'char','*',NULL, 1,0,1 , 2), +( 'caom2.ArtifactDescription', 'id', 'primary key', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 1,1,1 , 3), +( 'caom2.ArtifactDescription', 'metaProducer', 'identifier for the producer of this entity metadata (URI of the form {organisation}:{software}-{version}) [new in 2.4]', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 4), +( 'caom2.ArtifactDescription', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*', 'timestamp', 0,0,0, 5), +( 'caom2.ArtifactDescription', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 6) ; insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,unit,datatype,arraysize,xtype,principal,indexed,std,column_index) values -( 'caom2.DeletedObservationEvent', 'uri', 'URI of the deleted Observation', 'caom2:DeletedObservationEvent.uri', NULL, NULL, 'char','*','uri', 0,1,0 , 1), -( 'caom2.DeletedObservationEvent', 'id', 'primary key == the uuid of the deleted entity', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 0,1,0 , 2), -( 'caom2.DeletedObservationEvent', 'metaProducer', 'identifier for the producer of this entity; usually null in practice', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 3), -( 'caom2.DeletedObservationEvent', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*', 'timestamp', 1,1,0, 4), -( 'caom2.DeletedObservationEvent', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 5) +( 'caom2.DeletedObservationEvent', 'uri', 'URI of the deleted Observation', 'caom2:DeletedObservationEvent.uri', NULL, NULL, 'char','*','uri', 1,1,1 , 1), +( 'caom2.DeletedObservationEvent', 'id', 'primary key == the uuid of the deleted entity', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 1,1,1 , 2), +( 'caom2.DeletedObservationEvent', 'metaProducer', 'identifier for the producer of this entity; usually null in practice', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 3), +( 'caom2.DeletedObservationEvent', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*', 'timestamp', 0,0,0, 4), +( 'caom2.DeletedObservationEvent', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 5) ; insert into tap_schema.columns11 (table_name,column_name,description,utype,ucd,unit,datatype,arraysize,xtype,principal,indexed,std,column_index) values -( 'caom2.DeletedArtifactDescriptionEvent', 'uri', 'URI of the deleted ArtifactDescription', 'caom2:DeletedObservationEvent.uri', NULL, NULL, 'char','*','uri', 0,1,0 , 1), -( 'caom2.DeletedArtifactDescriptionEvent', 'id', 'primary key == the uuid of the deleted entity', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 0,1,0 , 2), -( 'caom2.DeletedArtifactDescriptionEvent', 'metaProducer', 'identifier for the producer of this entity; usually null in practice', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 3), -( 'caom2.DeletedArtifactDescriptionEvent', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*', 'timestamp', 1,1,0, 4), -( 'caom2.DeletedArtifactDescriptionEvent', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 1,0,0 , 5) +( 'caom2.DeletedArtifactDescriptionEvent', 'uri', 'URI of the deleted ArtifactDescription', 'caom2:DeletedObservationEvent.uri', NULL, NULL, 'char','*','uri', 1,1,1 , 1), +( 'caom2.DeletedArtifactDescriptionEvent', 'id', 'primary key == the uuid of the deleted entity', 'caom2:Entity.id', NULL, NULL, 'char','36','uuid', 1,1,1 , 2), +( 'caom2.DeletedArtifactDescriptionEvent', 'metaProducer', 'identifier for the producer of this entity; usually null in practice', 'caom2:Entity.metaProducer', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 3), +( 'caom2.DeletedArtifactDescriptionEvent', 'lastModified', 'timestamp of last modification of this row', 'caom2:Entity.lastModified', NULL, NULL, 'char', '23*', 'timestamp', 0,0,0, 4), +( 'caom2.DeletedArtifactDescriptionEvent', 'metaChecksum', 'checksum of the metadata in this entity (URI of the form {algorithm}:{hex value})', 'caom2:Entity.metaChecksum', NULL, NULL, 'char', '*', 'uri', 0,0,0 , 5) ; -- join tables @@ -295,19 +295,22 @@ insert into tap_schema.tables11 (schema_name,table_name,table_type,description) ('caom2', 'caom2.HarvestSkip', 'table', 'list of observations to retry or artifacts to sync'); insert into tap_schema.columns11 (table_name,column_name,utype,description,unit,datatype,arraysize,xtype,principal,indexed,std,column_index) values -( 'caom2.HarvestState', 'source', 'caom2:HarvestState.source', 'ID of the remote luskan (URI)', NULL, 'char','512*', 'uri', 1, 1, 1, 1 ), -( 'caom2.HarvestState', 'cname', 'caom2:HarvestState.cname', 'classname of the harvested entity', NULL, 'char','64*', NULL, 1, 1, 1, 2 ), -( 'caom2.HarvestState', 'curLastModified', 'caom2:HarvestState.curLastModified', 'lastModified timestamp of the last entity harvested', NULL, 'char','23*','timestamp', 1, 1, 1, 3 ), -( 'caom2.HarvestState', 'curID', 'caom2:HarvestState`.curID', 'id of the last entity harvested', NULL, 'char','36','uuid', 1, 1, 1, 4 ), -( 'caom2.HarvestState', 'lastModified', 'caom2:HarvestState.lastModified', 'timestamp of the event', NULL, 'char','23*','timestamp', 1, 1, 1, 5 ), -( 'caom2.HarvestState', 'stateID', 'caom2:HarvestState`.stateID', 'primary key', NULL, 'char','36','uuid', 1, 1, 1, 6 ); +( 'caom2.HarvestState', 'source', 'caom2:HarvestState.source', 'ID of the remote luskan (URI)', NULL, 'char','512*', 'uri', 1, 0, 1, 1 ), +( 'caom2.HarvestState', 'cname', 'caom2:HarvestState.cname', 'classname of the harvested entity', NULL, 'char','64*', NULL, 1, 0, 1, 2 ), +( 'caom2.HarvestState', 'curLastModified', 'caom2:HarvestState.curLastModified', 'lastModified timestamp of the last entity harvested', NULL, 'char','23*','timestamp', 1, 0, 1, 3 ), +( 'caom2.HarvestState', 'curID', 'caom2:HarvestState`.curID', 'id of the last entity harvested', NULL, 'char','36','uuid', 1, 0, 1, 4 ), + +( 'caom2.HarvestState', 'id', 'caom2:HarvestState`.stateID', 'primary key', NULL, 'char','36','uuid', 1, 1, 1, 5 ), +( 'caom2.HarvestState', 'lastModified', 'caom2:HarvestState.lastModified', 'timestamp of the event', NULL, 'char','23*','timestamp', 1, 0, 1, 6 ) +; insert into tap_schema.columns11 (table_name,column_name,description,datatype,arraysize,xtype,principal,indexed,std) values -( 'caom2.HarvestSkip', 'source', 'harvesting source', 'char', '*', NULL, 1,1,1), -( 'caom2.HarvestSkip', 'cname', 'entity (class name)', 'char', '*', NULL, 1,1,1), -( 'caom2.HarvestSkip', 'bucket', 'random bucket code', 'char', '3', NULL, 1,1,1), -( 'caom2.HarvestSkip', 'skipID', 'URI of skipped entity instance', 'char', '*', 'uri', 1,1,1), -( 'caom2.HarvestSkip', 'tryAfter', 'timestamp for next (re)try', 'char', '23*', 'timestamp', 1,1,1), +( 'caom2.HarvestSkip', 'source', 'harvesting source', 'char', '*', NULL, 1,0,1), +( 'caom2.HarvestSkip', 'cname', 'entity (class name)', 'char', '*', NULL, 1,0,1), +( 'caom2.HarvestSkip', 'uri', 'URI of skipped entity instance', 'char', '*', 'uri', 1,0,1), +( 'caom2.HarvestSkip', 'uriBucket', 'random bucket code', 'char', '3', NULL, 1,1,1), +( 'caom2.HarvestSkip', 'tryAfter', 'timestamp for next (re)try', 'char', '23*', 'timestamp', 1,0,1), ( 'caom2.HarvestSkip', 'errorMessage', 'reason for harvest failure', 'char', '*', NULL, 1,0,1), -( 'caom2.HarvestSkip', 'lastModified', 'last modification of this entry', 'char', '23*', 'timestamp', 1,0,1), -( 'caom2.HarvestSkip', 'id', 'primary key', 'char', '36', 'uuid', 1,1,1); +( 'caom2.HarvestSkip', 'id', 'primary key', 'char', '36', 'uuid', 1,1,1), +( 'caom2.HarvestSkip', 'lastModified', 'last modification of this entry', 'char', '23*', 'timestamp', 1,0,1) +; diff --git a/argus/src/main/resources/sql/ivoa.tap_schema_content11.sql b/argus/src/main/resources/sql/ivoa.tap_schema_content11.sql index ec432521d..06cfaa522 100644 --- a/argus/src/main/resources/sql/ivoa.tap_schema_content11.sql +++ b/argus/src/main/resources/sql/ivoa.tap_schema_content11.sql @@ -42,16 +42,16 @@ insert into tap_schema.tables11 (schema_name,table_name,table_type,description,t insert into tap_schema.columns11 (table_name,column_name,utype,description,ucd,unit,datatype,arraysize,xtype,principal,indexed,std,column_index,column_id) values ( 'ivoa.ObsCore', 'obs_publisher_did', 'obscore:Curation.PublisherDID', - 'publisher dataset identifier', 'meta.ref.uri;meta.curation', NULL, 'char', '256*','uri', 1,1,1,1, 'ivoaPublisherID'); + 'publisher dataset identifier', 'meta.ref.ivoid', NULL, 'char', '256*','uri', 1,1,1,1, 'ivoaPublisherID'); insert into tap_schema.columns11 (table_name,column_name,utype,description,ucd,unit,datatype,arraysize,xtype,principal,indexed,std, column_index) values ( 'ivoa.ObsCore', 'obs_collection', 'obscore:DataID.Collection', - 'short name for the data colection', 'meta.id', NULL, 'char','128*',NULL, 1,0,1,2), + 'short name for the data colection', 'meta.id', NULL, 'char','128*',NULL, 1,1,1,2), ( 'ivoa.ObsCore', 'facility_name', 'obscore:Provenance.ObsConfig.Facility.name', - 'telescope name', 'meta.id;instr.tel', NULL, 'char','128*',NULL, 1,0,1,3), + 'telescope name', 'meta.id;instr.tel', NULL, 'char','128*',NULL, 1,1,1,3), ( 'ivoa.ObsCore', 'instrument_name', 'obscore:Provenance.ObsConfig.Instrument.name', - 'instrument name', 'meta.id;instr', NULL, 'char','128*',NULL, 1,0,1,4), + 'instrument name', 'meta.id;instr', NULL, 'char','128*',NULL, 1,1,1,4), ( 'ivoa.ObsCore', 'target_name', 'obscore:Target.Name', 'name of intended target', 'meta.id;src', NULL, 'char', '32*',NULL, 1,0,1,5), ( 'ivoa.ObsCore', 'obs_id', 'obscore:DataID.observationID', @@ -88,9 +88,9 @@ insert into tap_schema.columns11 'region bounded by observation', 'pos.outline;obs.field', NULL, 'char','*','shape', 1,1,1,44), ( 'ivoa.ObsCore', 's_resolution', 'obscore:Char.SpatialAxis.Resolution.refval.value', 'typical spatial resolution', 'pos.angResolution', 'arcsec', 'double',NULL,NULL, 1,0,1,45), -( 'ivoa.ObsCore', 's_resolution_min', '???', +( 'ivoa.ObsCore', 's_resolution_min', 'obscore:Char.SpatialAxis.Resolution.Bounds.Limits.LoLimit', 'smallest spatial resolution', 'pos.angResolution;stat.min', 'arcsec', 'double',NULL,NULL, 1,0,1,46), -( 'ivoa.ObsCore', 's_resolution_max', '???', +( 'ivoa.ObsCore', 's_resolution_max', 'obscore:Char.SpatialAxis.Resolution.Bounds.Limits.HiLimit', 'largest spatial resolution', 'pos.angResolution;stat.max', 'arcsec', 'double',NULL,NULL, 1,0,1,47), ( 'ivoa.ObsCore', 's_xel1', 'obscore:Char.SpatialAxis.numBins1', 'dimensions (number of pixels) along one spatial axis', 'meta.number', NULL, 'long',NULL,NULL, 1,0,1,48), @@ -100,13 +100,13 @@ insert into tap_schema.columns11 insert into tap_schema.columns11 (table_name,column_name,utype,description,ucd,unit,datatype,arraysize,xtype,principal,indexed,std, column_index) values ( 'ivoa.ObsCore', 'em_min', 'obscore:Char.SpectralAxis.Coverage.Bounds.Limits.LoLimit', - 'start spectral coordinate value', 'em.wl;stat.min', 'm', 'double',NULL,NULL, 1,1,1,50), + 'start spectral coordinate value', 'em.wl;stat.min', 'm', 'double',NULL,NULL, 1,0,1,50), ( 'ivoa.ObsCore', 'em_max', 'obscore:Char.SpectralAxis.Coverage.Bounds.Limits.HiLimit', - 'stop spectral coordinate value', 'em.wl;stat.max', 'm', 'double',NULL,NULL, 1,1,1,51), + 'stop spectral coordinate value', 'em.wl;stat.max', 'm', 'double',NULL,NULL, 1,0,1,51), ( 'ivoa.ObsCore', 'em_res_power', 'obscore:Char.SpectralAxis.Resolution.ResolPower.refval', 'typical spectral resolution', 'spect.resolution', NULL, 'double',NULL,NULL, 1,0,1,52), -( 'ivoa.ObsCore', 'em_resolution', '???', - 'typical spectral resolution', 'spect.resolution', NULL, 'double',NULL,NULL, 1,0,1,53), +( 'ivoa.ObsCore', 'em_resolution', 'obscore:Char.SpectralAxis.Resolution.refval.value', + 'typical spectral resolution', 'spect.resolution;stat.mean', 'm', 'double',NULL,NULL, 1,0,1,53), ( 'ivoa.ObsCore', 'em_resolution_min', '???', 'minimum spectral resolution', 'spect.resolution;stat.min', NULL, 'double',NULL,NULL, 1,0,1,54), ( 'ivoa.ObsCore', 'em_resolution_max', '???', @@ -119,9 +119,9 @@ insert into tap_schema.columns11 insert into tap_schema.columns11 (table_name,column_name,utype,description,ucd,unit,datatype,arraysize,xtype,principal,indexed,std, column_index) values ( 'ivoa.ObsCore', 't_min', 'obscore:Char.TimeAxis.Coverage.Bounds.Limits.StartTime', - 'start time of observation (MJD)', 'time.start;obs.exposure', 'd', 'double',NULL,NULL, 1,1,1,60), + 'start time of observation (MJD)', 'time.start;obs.exposure', 'd', 'double',NULL,NULL, 1,0,1,60), ( 'ivoa.ObsCore', 't_max', 'obscore:Char.TimeAxis.Coverage.Bounds.Limits.StopTime', - 'end time of observation (MJD)', 'time.end;obs.exposure', 'd', 'double',NULL,NULL, 1,1,1,61), + 'end time of observation (MJD)', 'time.end;obs.exposure', 'd', 'double',NULL,NULL, 1,0,1,61), ( 'ivoa.ObsCore', 't_exptime', 'obscore:Char.TimeAxis.Coverage.Support.Extent', 'typical exposure time', 'time.duration;obs.exposure', 's', 'double',NULL,NULL, 1,1,1,62), ( 'ivoa.ObsCore', 't_exptime_min', '???', @@ -136,7 +136,7 @@ insert into tap_schema.columns11 insert into tap_schema.columns11 (table_name,column_name,utype,description,ucd,unit,datatype,arraysize,xtype,principal,indexed,std, column_index) values ( 'ivoa.ObsCore', 'pol_states', 'obscore:Char.PolarizationAxis.stateList', - 'polarization states present in the data', 'meta.code;phys.polarization', NULL, 'char','32*',NULL, 1,0,1,70), + 'polarization states present in the data', 'meta.code;phys.polarization', NULL, 'char','32*',NULL, 1,1,1,70), ( 'ivoa.ObsCore', 'pol_xel', 'obscore:Char.PolarizationAxis.numBins', 'dimensions (number of pixels) along the polarization axis', 'meta.number', NULL, 'long',NULL,NULL, 1,0,1,71), ( 'ivoa.ObsCore', 'o_ucd', 'obscore:Char.ObservableAxis.ucd', @@ -163,7 +163,7 @@ insert into tap_schema.keys11 (key_id,from_table,target_table,description) value ('ivoa-core-radio', 'ivoa.ObsCore_radio', 'ivoa.ObsCore','standard way to join ivoa.ObsCore and ivoa.ObsCore_radio'); insert into tap_schema.key_columns11 (key_id,from_column,target_column) values -('ivoa-core-radio', 'obs_publisher_id', 'obs_publisher_id'); +('ivoa-core-radio', 'obs_publisher_did', 'obs_publisher_did'); -- backwards compatible: fill "size" column with values from arraysize set above -- where arraysize is a possibly variable-length 1-dimensional value diff --git a/argus/src/main/webapp/capabilities.xml b/argus/src/main/webapp/capabilities.xml index c42229b9e..7af55c8d5 100644 --- a/argus/src/main/webapp/capabilities.xml +++ b/argus/src/main/webapp/capabilities.xml @@ -30,6 +30,12 @@ + + + https://replace.me.com/argus/examples + + + diff --git a/argus/src/main/webapp/examples/index.html b/argus/src/main/webapp/examples/index.html new file mode 100644 index 000000000..f5a38dc9e --- /dev/null +++ b/argus/src/main/webapp/examples/index.html @@ -0,0 +1,102 @@ + + + + CAOM Query Examples + + +
+
+

Standard CAOM Discovery Query with Spatial Constraint

+

+ standardID: ivo://ivoa.net/std/TAP +

+ +

+ tables: + caom2.Observation + JOIN + caom2.Plane +

+ +

parameters:

+
+ LANG = ADQL +
+ +
+ MAXREC = 100 +
+ +
+ QUERY = + 
+                    SELECT p.publisherID, o.telescope_name, o.instrument_name, 
+                        p.calibrationLevel, p.dataProductType, 
+                        p.position_bounds, p.energy_bounds, p.time_bounds, p.polarization_states
+                    FROM caom2.Observation AS o JOIN caom2.Plane AS p ON o.obsID = p.obsID
+                    WHERE o.intent = 'science' AND INTERSECTS(p.position_bounds, CIRCLE('ICRS', 180.0, 0.0, 0.5)) = 1
+
+
+

+ The Plane.publisherID column is the identifier that is needed to call the CAOM DataLink service and initiate + data access; when this column is selected, the service will include a service descriptor + with the necessary information. +

+
+ +
+

Standard CAOM Query with Primary Artifact

+

+ standardID: ivo://ivoa.net/std/TAP +

+ + +

+ tables: + caom2.Observation + JOIN + caom2.Plane + JOIN + caom2.Artifact +

+ +

parameters:

+
+ LANG = ADQL +
+ +
+ MAXREC = 100 +
+ +
+ QUERY = + 
+                    SELECT p.publisherID, o.telescope_name, o.instrument_name, 
+                        p.calibrationLevel, p.dataProductType, 
+                        p.position_bounds, p.energy_bounds, p.time_bounds, p.polarization_states,
+                        a.uri
+                    FROM caom2.Observation AS o
+                    JOIN caom2.Plane AS p ON o.obsID = p.obsID
+                    JOIN caom2.Artifact AS a ON p.planeID = a.planeID
+                    WHERE o.intent = 'science'
+                      AND a.productType IN ('science', 'this')
+                      AND INTERSECTS(p.position_bounds, CIRCLE('ICRS', 180.0, 0.0, 0.5)) = 1
+
+
+ +

+ The Plane.publisherID column is the identifier that is needed to call the CAOM DataLink service + and initiate data access; when this column is selected, the service will include a + service descriptor with the necessary information. +

+

+ Coming soon: The Artifact.uri column is the identifier that is needed to directly access + the file in the CADC Archive Storage system; when this column is selected, the service will + include a service descriptor with the necessary information. +

+
+ +
+ + diff --git a/caom2-db/src/intTest/java/org/opencadc/caom2/db/version/InitDatabaseTest.java b/caom2-db/src/intTest/java/org/opencadc/caom2/db/version/InitDatabaseTest.java index 6c696bbf1..a1cfa1e47 100644 --- a/caom2-db/src/intTest/java/org/opencadc/caom2/db/version/InitDatabaseTest.java +++ b/caom2-db/src/intTest/java/org/opencadc/caom2/db/version/InitDatabaseTest.java @@ -90,8 +90,8 @@ public class InitDatabaseTest { static String schema = "caom2"; static { - Log4jInit.setLevel("ca.nrc.cadc.caom2.version", Level.INFO); - Log4jInit.setLevel("ca.nrc.cadc.db.version", Level.INFO); + Log4jInit.setLevel("org.opencadc.caom2", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.db", Level.INFO); } private DataSource dataSource; diff --git a/caom2-db/src/main/java/org/opencadc/caom2/db/AbstractDAO.java b/caom2-db/src/main/java/org/opencadc/caom2/db/AbstractDAO.java index 046a78308..f8a0425a7 100644 --- a/caom2-db/src/main/java/org/opencadc/caom2/db/AbstractDAO.java +++ b/caom2-db/src/main/java/org/opencadc/caom2/db/AbstractDAO.java @@ -180,7 +180,7 @@ public void setConfig(Map config) { } try { this.dataSource = DBUtil.findJNDIDataSource(jndiDataSourceName); - log.warn("found: " + jndiDataSourceName); + log.debug("found: " + jndiDataSourceName); } catch (NamingException ex) { throw new IllegalArgumentException("cannot find JNDI DataSource: " + jndiDataSourceName); } diff --git a/caom2-db/src/main/java/org/opencadc/caom2/db/DeletedObservationEventIteratorQuery.java b/caom2-db/src/main/java/org/opencadc/caom2/db/DeletedObservationEventIteratorQuery.java index dd28cd461..d77d12389 100644 --- a/caom2-db/src/main/java/org/opencadc/caom2/db/DeletedObservationEventIteratorQuery.java +++ b/caom2-db/src/main/java/org/opencadc/caom2/db/DeletedObservationEventIteratorQuery.java @@ -138,7 +138,7 @@ public ResourceIterator query(DataSource ds) { } String sql = sb.toString(); - log.warn("SQL: " + sb.toString()); + log.debug("SQL: " + sb.toString()); Calendar utc = Calendar.getInstance(DateUtil.UTC); try { diff --git a/caom2-db/src/main/java/org/opencadc/caom2/db/ObservationStateIteratorQuery.java b/caom2-db/src/main/java/org/opencadc/caom2/db/ObservationStateIteratorQuery.java index d6efd24fa..a081c3513 100644 --- a/caom2-db/src/main/java/org/opencadc/caom2/db/ObservationStateIteratorQuery.java +++ b/caom2-db/src/main/java/org/opencadc/caom2/db/ObservationStateIteratorQuery.java @@ -90,6 +90,7 @@ public class ObservationStateIteratorQuery { private static final Logger log = Logger.getLogger(ObservationStateIteratorQuery.class); private final SQLGenerator gen; + private final SQLDialect dbDialect; private final String collection; private final String namespace; @@ -103,6 +104,7 @@ public ObservationStateIteratorQuery(SQLGenerator gen, String collection, String this.gen = gen; this.collection = collection; this.namespace = namespace; + this.dbDialect = gen.getDbDialect(); } public void setUriBucket(String uriBucket) { @@ -156,7 +158,7 @@ public ResourceIterator query(DataSource ds) { } String sql = sb.toString(); - log.warn("SQL: " + sb.toString()); + log.debug("SQL: " + sb.toString()); Calendar utc = Calendar.getInstance(DateUtil.UTC); try { @@ -200,7 +202,7 @@ private class ObservationStateIterator implements ResourceIterator getObservationStateExtractor() { - return new ObservationStateExtractor(); + return new ObservationStateExtractor(dbDialect); } public ResultSetExtractor getSkeletonExtractor(Class c) { diff --git a/caom2-db/src/main/java/org/opencadc/caom2/db/mappers/ObservationStateExtractor.java b/caom2-db/src/main/java/org/opencadc/caom2/db/mappers/ObservationStateExtractor.java index 37246e1ff..998356745 100644 --- a/caom2-db/src/main/java/org/opencadc/caom2/db/mappers/ObservationStateExtractor.java +++ b/caom2-db/src/main/java/org/opencadc/caom2/db/mappers/ObservationStateExtractor.java @@ -69,6 +69,7 @@ import java.sql.ResultSet; import java.sql.SQLException; +import org.opencadc.caom2.db.SQLDialect; import org.opencadc.caom2.util.ObservationState; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.ResultSetExtractor; @@ -79,14 +80,16 @@ */ public class ObservationStateExtractor implements ResultSetExtractor { - public ObservationStateExtractor() { + private ObservationStateMapper mapper; + + public ObservationStateExtractor(SQLDialect dbDialect) { + this.mapper = new ObservationStateMapper(dbDialect); } @Override public ObservationState extractData(ResultSet rs) throws SQLException, DataAccessException { if (rs.next()) { - ObservationStateMapper m = new ObservationStateMapper(); - return m.mapRow(rs, 1); + return mapper.mapRow(rs, 1); } return null; } diff --git a/caom2-db/src/main/java/org/opencadc/caom2/db/mappers/ObservationStateMapper.java b/caom2-db/src/main/java/org/opencadc/caom2/db/mappers/ObservationStateMapper.java index ed013aed2..ae30f0a6b 100644 --- a/caom2-db/src/main/java/org/opencadc/caom2/db/mappers/ObservationStateMapper.java +++ b/caom2-db/src/main/java/org/opencadc/caom2/db/mappers/ObservationStateMapper.java @@ -74,33 +74,44 @@ import java.util.Calendar; import java.util.Date; import java.util.UUID; +import org.opencadc.caom2.db.SQLDialect; import org.opencadc.caom2.db.Util; import org.opencadc.caom2.util.CaomUtil; import org.opencadc.caom2.util.ObservationState; import org.springframework.jdbc.core.RowMapper; /** - * + * Custom mapper to extract a subset of Observation fields. + * * @author pdowler */ public class ObservationStateMapper implements RowMapper { private final Calendar utcCalendar = Calendar.getInstance(DateUtil.UTC); + private final SQLDialect dbDialect; + public static final String[] COLUMNS = new String[] { - "uri", "maxLastModified", "accMetaChecksum", - "obsID" // PK + "obsID", + "uri", "maxLastModified", "accMetaChecksum", + "metaRelease", "metaReadGroups" }; - public ObservationStateMapper() { + public ObservationStateMapper(SQLDialect dbDialect) { + this.dbDialect = dbDialect; } public ObservationState mapRow(ResultSet rs, int i) throws SQLException { int col = 1; + UUID id = Util.getUUID(rs, col++); URI uri = Util.getURI(rs, col++); Date maxLastModified = Util.getDate(rs, col++, utcCalendar); URI accMetaChecksum = Util.getURI(rs, col++); - UUID id = Util.getUUID(rs, col++); - return new ObservationState(id, uri, maxLastModified, accMetaChecksum); + ObservationState ret = new ObservationState(id, uri, maxLastModified, accMetaChecksum); + + ret.metaRelease = Util.getDate(rs, col++, utcCalendar); + dbDialect.extractMultiURI(rs, col++, ret.getMetaReadGroups()); + + return ret; } } diff --git a/caom2-db/src/main/java/org/opencadc/caom2/db/version/InitDatabase.java b/caom2-db/src/main/java/org/opencadc/caom2/db/version/InitDatabase.java index 2f47eeb46..b8d65341c 100644 --- a/caom2-db/src/main/java/org/opencadc/caom2/db/version/InitDatabase.java +++ b/caom2-db/src/main/java/org/opencadc/caom2/db/version/InitDatabase.java @@ -84,8 +84,8 @@ public class InitDatabase extends ca.nrc.cadc.db.version.InitDatabase { public static final String MODEL_NAME = "CAOM"; // alpha: just core columns needed to round-trip // beta: minimal query optimization columns to support argus (TAP) and ObsCore - public static final String MODEL_VERSION = "2.5.0-beta"; - public static final String PREV_MODEL_VERSION = "2.5.0-alpha"; + public static final String MODEL_VERSION = "2.5.0-beta-2"; + public static final String PREV_MODEL_VERSION = "2.5.0-beta"; static String[] CREATE_SQL = new String[]{ "caom2.ModelVersion.sql", @@ -99,18 +99,19 @@ public class InitDatabase extends ca.nrc.cadc.db.version.InitDatabase { "caom2.HarvestSkip.sql", "caom2.DeletedObservationEvent.sql", "caom2.DeletedArtifactDescriptionEvent.sql", - //"caom2.extra_indices.sql", + "caom2.extra_indices.sql", "caom2.ObsCore.sql", "caom2.ObsCore_radio.sql", //"caom2.SIAv1.sql", "caom2.permissions.sql" }; - static String[] UPGRADE_SQL = new String[]{ - "caom2.upgrade-2.5.0-beta.sql", + static String[] UPGRADE_SQL = new String[] { + "caom2.upgrade-2.5.0-beta-2.sql", + "caom2.extra_indices.sql", "caom2.ObsCore.sql", "caom2.ObsCore_radio.sql", - "caom2.permissions.sql" + "caom2.permissions.sql", }; public InitDatabase(DataSource dataSource, String database, String schema) { diff --git a/caom2-db/src/main/resources/postgresql/caom2.Artifact.sql b/caom2-db/src/main/resources/postgresql/caom2.Artifact.sql index 77d046349..d540da88f 100644 --- a/caom2-db/src/main/resources/postgresql/caom2.Artifact.sql +++ b/caom2-db/src/main/resources/postgresql/caom2.Artifact.sql @@ -25,15 +25,15 @@ create table .Artifact ) ; --- this is for Plane join Artifact -create index i_planeID on .Artifact (planeID) +-- this is for Plane join Artifact; naming style same as pkey +create index artifact_fkey on .Artifact (planeID) ; -- tag the clustering index -cluster i_planeID on .Artifact +cluster artifact_fkey on .Artifact ; -create index Artifact_i_uri +create index i_artifact_uri on .Artifact (uri) ; diff --git a/caom2-db/src/main/resources/postgresql/caom2.ArtifactDescription.sql b/caom2-db/src/main/resources/postgresql/caom2.ArtifactDescription.sql index bc5412bf9..d9e12f988 100644 --- a/caom2-db/src/main/resources/postgresql/caom2.ArtifactDescription.sql +++ b/caom2-db/src/main/resources/postgresql/caom2.ArtifactDescription.sql @@ -13,7 +13,7 @@ create table .ArtifactDescription ; -- logical primary key -create unique index ArtifactDescription_i_uri +create unique index i_ArtifactDescription_uri on .ArtifactDescription (uri) ; diff --git a/caom2-db/src/main/resources/postgresql/caom2.Observation.sql b/caom2-db/src/main/resources/postgresql/caom2.Observation.sql index 0665fbb5b..aec54e0e4 100644 --- a/caom2-db/src/main/resources/postgresql/caom2.Observation.sql +++ b/caom2-db/src/main/resources/postgresql/caom2.Observation.sql @@ -71,7 +71,7 @@ create table .Observation ) ; -create unique index i_observationURI on .Observation (uri) +create unique index i_observation_uri on .Observation (uri) ; -- harvesting index diff --git a/caom2-db/src/main/resources/postgresql/caom2.Plane.sql b/caom2-db/src/main/resources/postgresql/caom2.Plane.sql index e436bc586..ca6893ab5 100644 --- a/caom2-db/src/main/resources/postgresql/caom2.Plane.sql +++ b/caom2-db/src/main/resources/postgresql/caom2.Plane.sql @@ -131,18 +131,18 @@ create table .Plane ) ; --- this is for Observation join Plane -create index i_obsID on .Plane (obsID) +-- this is for Observation join Plane; naming style same as pkey +create index plane_fkey on .Plane (obsID) ; -- tag the clustering index -cluster i_obsID on .Plane +cluster plane_fkey on .Plane ; -create unique index i_uri on .Plane(uri) +create unique index i_plane_uri on .Plane(uri) ; -create unique index i_publisherID on .Plane(publisherID) +create unique index i_plane_publisherID on .Plane(publisherID) ; -- join table diff --git a/caom2-db/src/main/resources/postgresql/caom2.extra_indices.sql b/caom2-db/src/main/resources/postgresql/caom2.extra_indices.sql index aac41a793..c5624ee05 100644 --- a/caom2-db/src/main/resources/postgresql/caom2.extra_indices.sql +++ b/caom2-db/src/main/resources/postgresql/caom2.extra_indices.sql @@ -1,106 +1,41 @@ --- position -create index Plane_position_i1 - on .Plane using gist (position_bounds_spoly) +-- core observation metadata +create index i_observation_collection + on .Observation (collection) ; - --- energy -create index Plane_energy_ib - on .Plane using gist (energy_bounds) +create index i_observation_telescope + on .Observation (telescope_name) ; - -create index Plane_energy_ib1 - on .Plane (energy_bounds_lower) +create index i_observation_instrument + on .Observation (instrument_name) ; -create index Plane_energy_ib2 - on .Plane (energy_bounds_upper) + +-- position +create index i_plane_position + on .Plane using gist (_q_position_bounds) + where _q_position_bounds is not null ; -create index Plane_energy_irw - on .Plane (energy_restwav) -where energy_restwav is not null +-- energy +create index i_plane_energy + on .Plane using gist (_q_energy_bounds) + where _q_energy_bounds is not null ; -- time -create index Plane_time_ib - on .Plane using gist (time_bounds) +create index i_plane_time + on .Plane using gist (_q_time_bounds) + where _q_time_bounds is not null ; -create index Plane_time_ib1 - on .Plane (time_bounds_lower) -; -create index Plane_time_ib2 - on .Plane (time_bounds_upper) -; -create index Plane_time_ibw - on .Plane (time_bounds_width) +create index i_plane_time_exposure + on .Plane (time_exposure) + where time_exposure is not null ; -create index plane_pol_states_pattern +-- polarization +create index _plane_polarization_states_pattern on .Plane (polarization_states varchar_pattern_ops) where polarization_states is not null ; -create index Observation_i_observationID_lower - on .Observation ( lower(observationID) ) -; -create index Observation_i_observationID_lower_pattern - on .Observation ( lower(observationID) varchar_pattern_ops) -; - -create index Observation_i_targ_lower - on .Observation ( lower(target_name) ) - where target_name is not null -; -create index Observation_i_targ_lower_pattern - on .Observation ( lower(target_name) varchar_pattern_ops) - where target_name is not null -; - -create index Observation_i_proposal_id_lower - on .Observation ( lower(proposal_id)) - where proposal_id is not null -; -create index Observation_i_proposal_id_lower_pattern - on .Observation ( lower(proposal_id) varchar_pattern_ops) - where proposal_id is not null -; - -create index i_collection_instrument - on .Observation (collection,instrument_name) -; -create index i_telescope_instrument - on .Observation (telescope_name,instrument_name) -; -create index i_instrument - on .Observation (instrument_name) -; -create index i_instrument_pattern - on .Observation (instrument_name varchar_pattern_ops) -; - -create index i_telescope - on .Observation (telescope_name) -; - -create index i_bandpassName - on .Plane (energy_bandpassName) - where energy_bandpassName is not null -; -create index i_bandpassName_pattern - on .Plane (energy_bandpassName varchar_pattern_ops) - where energy_bandpassName is not null -; - -create index i_provenance_runid - on .Plane (provenance_runID) - where provenance_runID is not null -; - -create index Plane_i_dataRelease - on .Plane ( dataRelease ) -; - -create index Artifact_i_uri_pattern - on .Artifact (uri varchar_pattern_ops) -; diff --git a/caom2-db/src/main/resources/postgresql/caom2.upgrade-2.5.0-beta-2.sql b/caom2-db/src/main/resources/postgresql/caom2.upgrade-2.5.0-beta-2.sql new file mode 100644 index 000000000..17105dcc2 --- /dev/null +++ b/caom2-db/src/main/resources/postgresql/caom2.upgrade-2.5.0-beta-2.sql @@ -0,0 +1,20 @@ +-- minor index naming tweaks for consistency +alter index .i_observationURI rename to i_observation_uri; + +alter index .i_uri rename to i_plane_uri; + +alter index .i_publisherID rename to i_plane_publisherID; + +alter index .Artifact_i_uri rename to i_artifact_uri; + +alter index .ArtifactDescription_i_uri rename to i_artifactdescription_uri; + + +-- foreign key indices +alter index .i_obsID rename to plane_fkey; +cluster plane_fkey on .Plane; + +alter index .i_planeID rename to artifact_fkey; +cluster artifact_fkey on .Artifact; + + diff --git a/caom2/src/main/java/org/opencadc/caom2/util/ObservationState.java b/caom2/src/main/java/org/opencadc/caom2/util/ObservationState.java index 013924bb9..33c369945 100644 --- a/caom2/src/main/java/org/opencadc/caom2/util/ObservationState.java +++ b/caom2/src/main/java/org/opencadc/caom2/util/ObservationState.java @@ -69,6 +69,8 @@ import java.net.URI; import java.util.Date; +import java.util.Set; +import java.util.TreeSet; import java.util.UUID; import org.apache.log4j.Logger; import org.opencadc.caom2.Observation; @@ -87,6 +89,10 @@ public class ObservationState { private final Date maxLastModified; private final URI accMetaChecksum; + // permission checking optimisation in torkeep + public Date metaRelease; + private final Set metaReadGroups = new TreeSet<>(); + // optional complete observation OR error public Observation observation; public Exception error; @@ -113,6 +119,10 @@ public URI getAccMetaChecksum() { public Date getMaxLastModified() { return maxLastModified; } + + public Set getMetaReadGroups() { + return metaReadGroups; + } @Override public String toString() { diff --git a/caom2/src/main/resources/CAOM-2.5.xsd b/caom2/src/main/resources/CAOM-2.5.xsd index afba88e9a..93ef6f275 100644 --- a/caom2/src/main/resources/CAOM-2.5.xsd +++ b/caom2/src/main/resources/CAOM-2.5.xsd @@ -709,6 +709,7 @@ + diff --git a/torkeep/README.md b/torkeep/README.md index a12b1a62b..5981e77d5 100644 --- a/torkeep/README.md +++ b/torkeep/README.md @@ -42,6 +42,13 @@ In addition, the TAP service does not currently support a configurable schema na See cadc-registry. +In addition to the OpenCADC permissions API (see _grantProvider_ below) `torkeep` can also be +configured to call the SRCNet Permissions API. This is configured along with the OpenID provider in `cadc-registry.properties`: +``` +ivo://skao.int/std/AuthAPI = https://.... +ivo://skao.int/std/PermissionsAPI = https://.... +``` +The keys here are like an IVOA standardID that identify an API feature; the value is the base URL of the service that provides the feature. ### torkeep.properties The torkeep.properties configures the services that provide grants to access collections, and collection specific properties. diff --git a/torkeep/VERSION b/torkeep/VERSION index c8b6d4d9e..aa9f8ca7f 100644 --- a/torkeep/VERSION +++ b/torkeep/VERSION @@ -1,7 +1,7 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VER=2.0-BETA-06 +VER=2.0-BETA-07 TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")" unset VER diff --git a/torkeep/build.gradle b/torkeep/build.gradle index 80c6db443..155c0d68d 100644 --- a/torkeep/build.gradle +++ b/torkeep/build.gradle @@ -29,7 +29,7 @@ dependencies { implementation 'net.sourceforge.javacsv:javacsv:2.0' - implementation 'org.opencadc:cadc-permissions-client:[0.3,1.0)' + implementation 'org.opencadc:cadc-permissions-client:[0.4.0,1.0)' implementation 'org.opencadc:caom2:[2.5.0,2.6)' implementation 'org.opencadc:cadc-util:[1.12.16,2.0)' implementation 'org.opencadc:cadc-rest:[1.4.7,)' diff --git a/torkeep/src/main/java/org/opencadc/torkeep/DeleteAction.java b/torkeep/src/main/java/org/opencadc/torkeep/DeleteAction.java index 21c90a48e..2fb81b46e 100644 --- a/torkeep/src/main/java/org/opencadc/torkeep/DeleteAction.java +++ b/torkeep/src/main/java/org/opencadc/torkeep/DeleteAction.java @@ -91,10 +91,12 @@ public DeleteAction() { @Override public void doAction() throws Exception { + checkWritable(); + URI uri = getObservationURI(); log.debug("START: " + uri); - checkWritePermission(); + checkWritePermission("DELETE"); ObservationDAO dao = getDAO(); ObservationState existing = dao.getState(uri); diff --git a/torkeep/src/main/java/org/opencadc/torkeep/GetAction.java b/torkeep/src/main/java/org/opencadc/torkeep/GetAction.java index 940f0f2b4..6c313902a 100644 --- a/torkeep/src/main/java/org/opencadc/torkeep/GetAction.java +++ b/torkeep/src/main/java/org/opencadc/torkeep/GetAction.java @@ -102,7 +102,8 @@ public GetAction() { @Override public void doAction() throws Exception { - log.debug("GET ACTION"); + checkReadable(); + URI uri = getObservationURI(); if (uri != null) { doGetObservation(uri); @@ -119,36 +120,38 @@ public void doAction() throws Exception { protected void doGetObservation(URI uri) throws Exception { log.debug("START: " + uri); - ObservationDAO dao = getDAO(); log.debug("getState: " + uri); ObservationState state = dao.getState(uri); log.debug("found state: " + state); - if (state == null) { - throw new ResourceNotFoundException("not found: " + uri); - } - - - Observation o = dao.get(state.getID()); - log.debug("loaded observation: " + o); // permission check: ignoring Observation.metaReadGroups for now Date now = new Date(); - if (o.metaRelease != null && o.metaRelease.before(now)) { + if (state != null && state.metaRelease != null && state.metaRelease.before(now)) { DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); - logInfo.setGrant("metaRelease: " + df.format(o.metaRelease)); + logInfo.setGrant("metaRelease: " + df.format(state.metaRelease)); } else { // check configured permissions checkReadPermission(); } - String mimeType = syncInput.getHeader("accept"); - syncOutput.setHeader("Content-Type", mimeType); - syncOutput.setHeader("ETag", o.getAccMetaChecksum()); + if (state == null) { + throw new ResourceNotFoundException("not found: " + uri); + } + + Observation o = dao.get(state.getID()); + log.debug("loaded observation: " + o); + + syncOutput.setHeader("x-entity-id", o.getID().toString()); + syncOutput.setHeader("etag", o.getAccMetaChecksum()); syncOutput.setLastModified(o.getMaxLastModified()); + String mimeType = syncInput.getHeader("accept"); + Output op = getObservationWriter(mimeType); + syncOutput.setHeader("Content-Type", op.contentType); + ObservationWriter ow = op.ow; + OutputStream os = syncOutput.getOutputStream(); ByteCountOutputStream bc = new ByteCountOutputStream(os); - ObservationWriter ow = getObservationWriter(mimeType); ow.write(o, bc); logInfo.setBytes(bc.getByteCount()); @@ -169,17 +172,27 @@ protected void doList(int maxRec, Date start, Date end, boolean isAscending) thr log.debug("DONE: " + getCollection()); } - private ObservationWriter getObservationWriter(String mt) throws UnsupportedOperationException { - ObservationWriter ret; + private class Output { + ObservationWriter ow; + String contentType; + } + + private Output getObservationWriter(String mt) throws UnsupportedOperationException { + Output ret = new Output(); if (JSON_MIMETYPE.equals(mt) || CAOM_JSON_MIMETYPE.equals(mt)) { - return new JsonWriter(true); + ret.ow = new JsonWriter(true); + ret.contentType = mt; } if (XML_MIMETYPE.equals(mt) || CAOM_XML_MIMETYPE.equals(mt)) { - return new ObservationWriter(); + ret.ow = new ObservationWriter(); + ret.contentType = mt; } // ignore unsupported format request // default: XML - return new ObservationWriter(); + ret.ow = new ObservationWriter(); + ret.contentType = CAOM_XML_MIMETYPE; + + return ret; } protected long writeObservationList(ResourceIterator iter) throws IOException { diff --git a/torkeep/src/main/java/org/opencadc/torkeep/HeadAction.java b/torkeep/src/main/java/org/opencadc/torkeep/HeadAction.java index c26990a49..76422726c 100644 --- a/torkeep/src/main/java/org/opencadc/torkeep/HeadAction.java +++ b/torkeep/src/main/java/org/opencadc/torkeep/HeadAction.java @@ -91,7 +91,10 @@ public HeadAction() { @Override public void doAction() throws Exception { + checkReadable(); + URI uri = getObservationURI(); + log.debug("START: " + uri); if (uri == null) { // TODO: method not supported on base endpoint @@ -101,26 +104,24 @@ public void doAction() throws Exception { log.debug("getState: " + uri); ObservationState state = dao.getState(uri); log.debug("found state: " + state); - if (state == null) { - throw new ResourceNotFoundException("not found: " + uri); - } - - Observation o = dao.get(state.getID()); - log.debug("loaded observation: " + o); // permission check: ignoring Observation.metaReadGroups for now Date now = new Date(); - if (o.metaRelease != null && o.metaRelease.before(now)) { + if (state != null && state.metaRelease != null && state.metaRelease.before(now)) { DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC); - logInfo.setGrant("metaRelease: " + df.format(o.metaRelease)); + logInfo.setGrant("metaRelease: " + df.format(state.metaRelease)); } else { // check configured permissions checkReadPermission(); } - syncOutput.setHeader("x-entity-id", o.getID().toString()); - syncOutput.setHeader("etag", o.getAccMetaChecksum()); - syncOutput.setLastModified(o.getMaxLastModified()); + if (state == null) { + throw new ResourceNotFoundException("not found: " + uri); + } + + syncOutput.setHeader("x-entity-id", state.getID().toString()); + syncOutput.setHeader("etag", state.getAccMetaChecksum()); + syncOutput.setLastModified(state.getMaxLastModified()); syncOutput.setCode(200); } } diff --git a/torkeep/src/main/java/org/opencadc/torkeep/PostAction.java b/torkeep/src/main/java/org/opencadc/torkeep/PostAction.java index 70d5b1542..799900d0e 100644 --- a/torkeep/src/main/java/org/opencadc/torkeep/PostAction.java +++ b/torkeep/src/main/java/org/opencadc/torkeep/PostAction.java @@ -92,10 +92,12 @@ public PostAction() { @Override public void doAction() throws Exception { + checkWritable(); + URI uri = getObservationURI(); log.debug("START: " + uri); - checkWritePermission(); + checkWritePermission("POST"); Observation obs = getInputObservation(); if (obs == null) { diff --git a/torkeep/src/main/java/org/opencadc/torkeep/PutAction.java b/torkeep/src/main/java/org/opencadc/torkeep/PutAction.java index ad83b65b6..054dfe723 100644 --- a/torkeep/src/main/java/org/opencadc/torkeep/PutAction.java +++ b/torkeep/src/main/java/org/opencadc/torkeep/PutAction.java @@ -93,10 +93,12 @@ public PutAction() { @Override public void doAction() throws Exception { + checkWritable(); + URI uri = getObservationURI(); log.debug("START: " + uri); - checkWritePermission(); + checkWritePermission("PUT"); Observation obs = getInputObservation(); if (obs == null) { diff --git a/torkeep/src/main/java/org/opencadc/torkeep/RepoAction.java b/torkeep/src/main/java/org/opencadc/torkeep/RepoAction.java index acf20e8cc..a0c41384e 100644 --- a/torkeep/src/main/java/org/opencadc/torkeep/RepoAction.java +++ b/torkeep/src/main/java/org/opencadc/torkeep/RepoAction.java @@ -70,12 +70,15 @@ package org.opencadc.torkeep; import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.AuthorizationToken; import ca.nrc.cadc.caom2.compute.CaomWCSValidator; import ca.nrc.cadc.caom2.compute.ComputeUtil; import ca.nrc.cadc.date.DateUtil; import ca.nrc.cadc.io.ByteCountOutputStream; +import ca.nrc.cadc.net.PermissionDeniedException; import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.net.TransientException; +import ca.nrc.cadc.reg.client.LocalAuthority; import ca.nrc.cadc.rest.InlineContentHandler; import ca.nrc.cadc.rest.RestAction; import com.csvreader.CsvWriter; @@ -85,7 +88,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.security.AccessControlException; import java.security.Principal; import java.security.cert.CertificateException; import java.text.DateFormat; @@ -93,7 +95,6 @@ import java.util.Date; import java.util.Map; import javax.security.auth.Subject; -import javax.security.auth.x500.X500Principal; import org.apache.log4j.Logger; import org.opencadc.caom2.Artifact; import org.opencadc.caom2.Observation; @@ -103,6 +104,9 @@ import org.opencadc.caom2.util.CaomValidator; import org.opencadc.caom2.xml.ObservationParsingException; import org.opencadc.permissions.client.PermissionsCheck; +import org.opencadc.permissions.client.srcnet.AuthorisationResult; +import org.opencadc.permissions.client.srcnet.PAPI; +import org.opencadc.permissions.client.srcnet.PermissionsAPIClient; /** * @author pdowler @@ -110,6 +114,9 @@ public abstract class RepoAction extends RestAction { private static final Logger log = Logger.getLogger(RepoAction.class); + // copied from cavern and hard coded for now + private static final String PAPI_SRV = "science-metadata"; + static final String CAOM_XML_MIMETYPE = "application/x-caom+xml"; static final String XML_MIMETYPE = "text/xml"; @@ -331,20 +338,14 @@ protected Observation getInputObservation() throws ObservationParsingException { /** * Check if the caller can read the resource. * - * @throws AccessControlException + * @throws ca.nrc.cadc.net.PermissionDeniedException * @throws java.security.cert.CertificateException * @throws ca.nrc.cadc.net.ResourceNotFoundException * @throws java.io.IOException */ - protected void checkReadPermission() throws AccessControlException, + protected void checkReadPermission() throws PermissionDeniedException, CertificateException, ResourceNotFoundException, IOException { log.debug("check READ permission for collection: " + getCollection()); - if (!readable) { - if (!writable) { - throw new IllegalStateException(STATE_OFFLINE_MSG); - } - throw new IllegalStateException(STATE_READ_ONLY_MSG); - } // config for the collection TorkeepConfig tc = getTorkeepConfig(); @@ -364,7 +365,7 @@ protected void checkReadPermission() throws AccessControlException, grantURI = URI.create(base + getCollection() + "?"); } } - log.debug("authorizing: " + grantURI); + log.warn("authorizing: " + grantURI); Subject subject = AuthenticationUtil.getCurrentSubject(); boolean operators = !tc.archiveOperators.isEmpty() || !tc.metaSyncOperators.isEmpty(); @@ -397,30 +398,52 @@ protected void checkReadPermission() throws AccessControlException, } try { - PermissionsCheck cp = new PermissionsCheck(grantURI, false, logInfo); - cp.checkReadPermission(tc.getGrantProviders()); + if (!tc.getGrantProviders().isEmpty()) { + PermissionsCheck cp = new PermissionsCheck(grantURI, false, logInfo); + cp.checkReadPermission(tc.getGrantProviders()); + // PermissionsCheck either returns (if authorised) or throws + return; + } } catch (InterruptedException ex) { throw new RuntimeException("interrupted", ex); } + + LocalAuthority loc = new LocalAuthority(); + URI authAPI = loc.getResourceID(PAPI.STD_AUTH_API); + URI permAPI = loc.getResourceID(PAPI.STD_PERM_API); + AuthorizationToken tok = getAuthorizationToken(subject); + if (tok != null && authAPI != null && permAPI != null) { + String srv = PAPI_SRV; + String route = "/observations/" + syncInput.getPath(); + + log.warn("call papi: " + permAPI + " service=" + srv + " route=" + route + " method=GET"); + PermissionsAPIClient permissionsAPIClient = new PermissionsAPIClient(permAPI.toURL(), authAPI.toURL()); + AuthorisationResult authorisationResult = permissionsAPIClient.authoriseRoute( + srv, route, + tok.getCredentials(), // ignores token domains and scope + "GET", null, "1"); + log.warn("papi: authorised=" + authorisationResult.isAuthorised + " route=" + route); + if (authorisationResult.isAuthorised) { + logInfo.setResource(grantURI); + logInfo.setGrant("read: " + permAPI.toASCIIString()); + return; + } + } + + throw new PermissionDeniedException("permission denied"); } /** * Check if the caller can create or modify the resource. * - * @throws AccessControlException + * @throws ca.nrc.cadc.net.PermissionDeniedException * @throws java.security.cert.CertificateException * @throws ca.nrc.cadc.net.ResourceNotFoundException * @throws java.io.IOException */ - protected void checkWritePermission() throws AccessControlException, + protected void checkWritePermission(String method) throws PermissionDeniedException, CertificateException, ResourceNotFoundException, IOException { log.debug("check WRITE permission for collection: " + getCollection()); - if (!writable) { - if (readable) { - throw new IllegalStateException(STATE_READ_ONLY_MSG); - } - throw new IllegalStateException(STATE_OFFLINE_MSG); - } TorkeepConfig tc = getTorkeepConfig(); CollectionEntry entry = tc.getConfig(getCollection()); @@ -434,7 +457,7 @@ protected void checkWritePermission() throws AccessControlException, } else { grantURI = URI.create("caom:" + getCollection() + "/"); } - log.debug("authorizing: " + grantURI); + log.warn("authorizing: " + grantURI); Subject subject = AuthenticationUtil.getCurrentSubject(); boolean operators = !tc.archiveOperators.isEmpty(); @@ -446,7 +469,7 @@ protected void checkWritePermission() throws AccessControlException, for (Principal p : xa.getPrincipals()) { if (AuthenticationUtil.equals(cp, p)) { logInfo.setResource(grantURI); - logInfo.setGrant("read: archiveOperator"); + logInfo.setGrant("write: archiveOperator"); // granted return; } @@ -457,11 +480,38 @@ protected void checkWritePermission() throws AccessControlException, } try { - PermissionsCheck cp = new PermissionsCheck(grantURI, false, logInfo); - cp.checkWritePermission(tc.getGrantProviders()); + if (!tc.getGrantProviders().isEmpty()) { + PermissionsCheck cp = new PermissionsCheck(grantURI, false, logInfo); + cp.checkWritePermission(tc.getGrantProviders()); + // PermissionsCheck either returns (if authorised) or throws + return; + } } catch (InterruptedException ex) { throw new RuntimeException("interrupted", ex); } + + LocalAuthority loc = new LocalAuthority(); + URI authAPI = loc.getResourceID(PAPI.STD_AUTH_API); + URI permAPI = loc.getResourceID(PAPI.STD_PERM_API); + AuthorizationToken tok = getAuthorizationToken(subject); + if (tok != null && authAPI != null && permAPI != null) { + String srv = PAPI_SRV; + String route = "/observations/" + syncInput.getPath(); + log.warn("call papi: " + permAPI + " service=" + srv + " route=" + route + " method=" + method); + PermissionsAPIClient permissionsAPIClient = new PermissionsAPIClient(permAPI.toURL(), authAPI.toURL()); + AuthorisationResult authorisationResult = permissionsAPIClient.authoriseRoute( + srv, route, + tok.getCredentials(), // ignores token domains and scope + method, null, "1"); + log.warn("papi: authorised=" + authorisationResult.isAuthorised + " route=" + route); + if (authorisationResult.isAuthorised) { + logInfo.setResource(grantURI); + logInfo.setGrant("write: " + permAPI.toASCIIString()); + return; + } + } + + throw new PermissionDeniedException("permission denied"); } protected void assignPublisherID(Observation obs) throws URISyntaxException { @@ -496,7 +546,7 @@ static void assignPublisherID(Observation obs, String basePublisherID) throws UR } protected void validate(Observation obs) - throws AccessControlException, IOException, TransientException { + throws IOException, TransientException { try { CollectionEntry ce = torkeepConfig.getConfig(collection); ValidationPolicy validator = new ValidationPolicy(ce.getValidationPolicy()); @@ -554,4 +604,11 @@ protected void validate(Observation obs) protected InlineContentHandler getInlineContentHandler() { return null; } + + static AuthorizationToken getAuthorizationToken(final Subject subject) { + return subject.getPublicCredentials(AuthorizationToken.class).stream() + .filter(token -> AuthenticationUtil.CHALLENGE_TYPE_BEARER.equalsIgnoreCase(token.getType())) + .findFirst() + .orElse(null); + } } diff --git a/torkeep/src/main/webapp/WEB-INF/web.xml b/torkeep/src/main/webapp/WEB-INF/web.xml index b8461088b..5db161d9e 100644 --- a/torkeep/src/main/webapp/WEB-INF/web.xml +++ b/torkeep/src/main/webapp/WEB-INF/web.xml @@ -14,9 +14,10 @@ logLevelPackages org.opencadc.torkeep + org.opencadc.permissions + org.opencadc.caom2 ca.nrc.cadc.db.version ca.nrc.cadc.rest - ca.nrc.cadc.caom2 ca.nrc.cadc.wcs ca.nrc.cadc.vosi