diff --git a/backend/src/api-tests/species/dwcArchiveExport.test.ts b/backend/src/api-tests/species/dwcArchiveExport.test.ts index 7ec7a85c..de8352d8 100644 --- a/backend/src/api-tests/species/dwcArchiveExport.test.ts +++ b/backend/src/api-tests/species/dwcArchiveExport.test.ts @@ -58,6 +58,7 @@ describe('DwC-A species export (admin-only)', () => { const measurementCsv = await zip.file('measurementorfact.csv')!.async('string') expect(measurementCsv).toContain('"measurementID"') expect(measurementCsv).toContain('"verbatimMeasurementType"') + expect(measurementCsv).not.toContain('"measurementRemarks"') const metaXml = await zip.file('meta.xml')!.async('string') expect(metaXml).toContain(' type SpeciesForMeasurementExport = Pick< Prisma.com_species, | 'species_id' + | 'strain' + | 'gene' + | 'taxon_status' | 'body_mass' | 'brain_mass' + | 'sv_length' + | 'sd_size' + | 'sd_display' + | 'tshm' + | 'symph_mob' + | 'relative_blade_length' + | 'tht' | 'diet1' | 'diet2' | 'diet3' | 'diet_description' + | 'rel_fib' + | 'selectivity' + | 'digestion' + | 'feedinghab1' + | 'feedinghab2' + | 'shelterhab1' + | 'shelterhab2' | 'locomo1' | 'locomo2' | 'locomo3' + | 'hunt_forage' | 'activity' | 'crowntype' | 'microwear' + | 'horizodonty' + | 'cusp_shape' + | 'cusp_count_buccal' + | 'cusp_count_lingual' + | 'loph_count_lon' + | 'loph_count_trs' + | 'fct_al' + | 'fct_ol' + | 'fct_sf' + | 'fct_ot' + | 'fct_cm' | 'mesowear' + | 'mw_or_high' + | 'mw_or_low' + | 'mw_cs_sharp' + | 'mw_cs_round' + | 'mw_cs_blunt' + | 'mw_scale_min' + | 'mw_scale_max' | 'mw_value' + | 'pop_struc' + | 'sp_status' > const MEASUREMENT_FIELD_MAPPINGS: Array<{ @@ -246,6 +283,29 @@ const MEASUREMENT_FIELD_MAPPINGS: Array<{ measurementUnit: string measurementMethod: string }> = [ + // NOTE: In v1, measurementMethod is populated from the Pantheria VSP manual where available: + // https://www.pantherion.com/dbmanual97/VSP.html + { + field: 'strain', + measurementType: 'strain', + measurementUnit: '', + // TODO(#1150): Add field description / controlled vocabulary. + measurementMethod: '', + }, + { + field: 'gene', + measurementType: 'gene', + measurementUnit: '', + // TODO(#1150): Add field description / controlled vocabulary. + measurementMethod: '', + }, + { + field: 'taxon_status', + measurementType: 'taxon status', + measurementUnit: '', + // TODO(#1150): Add field description / controlled vocabulary. + measurementMethod: '', + }, { field: 'body_mass', measurementType: 'body mass', @@ -260,6 +320,53 @@ const MEASUREMENT_FIELD_MAPPINGS: Array<{ measurementMethod: 'The average adult brain mass estimated for the species, in grams. Where there is sexual dimorphism in size, put the mean of the two sexes here and record the masses per sex, if known, in the Comment field. Confidence intervals, if known, can also be put there.', }, + { + field: 'sv_length', + measurementType: 'snout-vent length', + measurementUnit: '', + measurementMethod: + 'For many species body-mass values will be unavailable or cannot be estimated with any confidence. However, every species should be classifiable into one of the gross size ranges listed below. This field will allow at least a crude characterization of body sizes for any fossil locality.', + }, + { + field: 'sd_size', + measurementType: 'sexual dimorphism - size', + measurementUnit: '', + measurementMethod: 'Whether there is sexual dimorphism in overall body size.', + }, + { + field: 'sd_display', + measurementType: 'sexual dimorphism - display', + measurementUnit: '', + measurementMethod: + 'Whether there is evidence of sexual dimorphism in display (or sexual combat) structures. (e. g., horns, antlers, dome-heads, canines). If the presence of these features is unknown, leave the field blank rather than enter "n."', + }, + { + field: 'tshm', + measurementType: 'tooth shape -- multicuspid', + measurementUnit: '', + measurementMethod: + 'A description of the morphology of the tooth crown, for multicusped teeth (if present). In concert with the other tooth morphology fields, this may allow functional interpretations to be made independently of whatever has been entered in the diet fields. Terminology for tooth-crown morphology is most highly developed for extant and fossil mammals, but no system has gained universal acceptance. The following reflects a compromise among many competing traditional systems, and is based partly on Fortelius (1985) and Janis and Fortelius (1988). This field is currently subject to further development. Improved nomenclature for some mammal groups, such as rodents and insectivores, might be more functionally indicative. Also, an expanded list of terms would be useful to characterize more fully the variation found among nonmammalian terrestrial vertebrates -- dinosaurs and therapsids in particular. The similar Molar Crown Type field is based on an alternative descriptive classification scheme, and currently applies only to mammals.', + }, + { + field: 'symph_mob', + measurementType: 'symphyseal mobility', + measurementUnit: '', + measurementMethod: 'Whether or not the mandibular symphysis is mobile.', + }, + { + field: 'relative_blade_length', + measurementType: 'relative blade length', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'tht', + measurementType: 'tooth height', + measurementUnit: '', + measurementMethod: + 'An indication of hypsodonty (tooth crown height) or the nature of other adaptations to deal with the problem of lifetime tooth wear. Tooth replacement, Tooth plates, and Hypselodont (ever-growing teeth) are absolute descriptors. The terms Brachydont, Mesodont and Hypsodont refer to different degrees of crown height of (mammalian) cheek teeth, and are subject to a variety of interpretations. Hypsodont (high-crowned) teeth may be defined objectively as those where the antero-posterior length is exceeded by the dorso-ventral height (Janis & Fortelius, 1988). "Somewhat hypsodont" teeth, intermediate between brachydont and hypsodont, are referred to as "mesodont," but there is no corresponding objective definition of this term. Quantitative indices of hypsodonty have been used (Janis, 1988), and might prove superior to the classification scheme presented here. Thus, this field is currently subject to further development.', + }, { field: 'diet1', measurementType: 'diet category 1', @@ -288,6 +395,55 @@ const MEASUREMENT_FIELD_MAPPINGS: Array<{ // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. measurementMethod: '', }, + { + field: 'rel_fib', + measurementType: 'relative fiber content', + measurementUnit: '', + measurementMethod: + 'The relative amount of plant fiber in the food of the species. Plant food can be divided into cell contents such as sugars, proteins and storage carbohydrates, which are directly digestible by vertebrates. Plant cell-walls, however, are composed of material ("fiber") partially digestible only by microbial fermentation. Thus, the higher the fiber content, relative to the amount of energy contained in the easily-digested portion, the harder it is to obtain energy from the forage and the poorer the "quality" of the food on a per-unit basis. In addition, the proportion of the fiber digestible by fermentation also varies among plant species, plant parts, and growth stages. This field describes the food as having high, medium, and low levels of fiber. It is intended as a rough indication of the nutritional quality of a species\' diet. It refers only to herbivorous diets, or the plant portions of omnivorous diets. (The field basically functions to group various Diet 3 categories by relative fiber content.)', + }, + { + field: 'selectivity', + measurementType: 'selectivity', + measurementUnit: '', + measurementMethod: + 'Within its food-type category (Diet 1-3) a species may feed selectively or unselectively. Thus this field applies to any dietary category. Some food types impose selectivity restrictions on the species that feed on them. For example, most large grazers are less selective than mixed feeders or browsers. This is not what this field is meant to indicate! Rather, it applies within dietary categories. It could, for example, be used to distinguish between relatively selective and relatively unselective grazers.', + }, + { + field: 'digestion', + measurementType: 'digestion', + measurementUnit: '', + measurementMethod: + 'There are different broad strategies for breaking down plant material by means of microbial activity in the gut. Hindgut fermenters (hg) and foregut fermenters (fg) are found in a variety of living taxa. True ruminants (ru) are confined to the ruminant artiodactyls; they are separated here from other foregut fermenters, of which they form a special derived subclass.', + }, + { + field: 'feedinghab1', + measurementType: 'feeding habitat 1', + measurementUnit: '', + measurementMethod: + 'The general habitat from which the species obtains the major part of its trophic resources, and in which it ordinarily spends time feeding. The allowed values are identical to those for Shelter Habitat 1. See also Feeding Habitat 2.', + }, + { + field: 'feedinghab2', + measurementType: 'feeding habitat 2', + measurementUnit: '', + measurementMethod: + 'For the Terrestrial (te) entry in Feeding Habitat 1 only, a further breakdown into more specific feeding habitats. They are described more fully below.', + }, + { + field: 'shelterhab1', + measurementType: 'shelter habitat 1', + measurementUnit: '', + measurementMethod: + 'The general habitat in which the animal sleeps, shelters, or avoids predation when not feeding. The allowed values are identical to those for Feeding Habitat 1. See also Shelter Habitat 2.', + }, + { + field: 'shelterhab2', + measurementType: 'shelter habitat 2', + measurementUnit: '', + measurementMethod: + 'For the Terrestrial (te) entry in Shelter Habitat 1 only, a further breakdown into more specific shelter habitats. They are described more fully below, and are mostly identical to the fields for Feeding Habitat 2.', + }, { field: 'locomo1', measurementType: 'locomotion 1', @@ -309,6 +465,13 @@ const MEASUREMENT_FIELD_MAPPINGS: Array<{ measurementMethod: 'The predominant mode of locomotor activity. [These categories are not necessarily complete at this time.] The categorization of flight locomotion in Locomotion 2 and Locomotion 3 is based on Norberg (1985).', }, + { + field: 'hunt_forage', + measurementType: 'hunt/forage', + measurementUnit: '', + measurementMethod: + 'The predominant hunting or foraging mode for carnivores. These categories are based upon those of Van Valkenburgh (1985) and are described more fully there. This field might also be of eventual use in describing foraging modes of non-carnivores, but at present these cannot be determined directly upon morphological criteria (such inferences as can be made are already taken care of in Feeding Habitat, Diet and Locomotion.)', + }, { field: 'activity', measurementType: 'activity', @@ -330,6 +493,83 @@ const MEASUREMENT_FIELD_MAPPINGS: Array<{ measurementMethod: "This field describes the kind of microwear (in terms of striations or pits) revealed by microscopic examination of the wear facets of the tooth crowns of the species. A considerable literature exists concerning the ways to infer aspects of a species' diet from patterns of microwear.", }, + { + field: 'horizodonty', + measurementType: 'horizodonty', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'cusp_shape', + measurementType: 'cusp shape', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'cusp_count_buccal', + measurementType: 'cusp count (buccal)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'cusp_count_lingual', + measurementType: 'cusp count (lingual)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'loph_count_lon', + measurementType: 'loph count (longitudinal)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'loph_count_trs', + measurementType: 'loph count (transverse)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'fct_al', + measurementType: 'functional crown type (AL)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'fct_ol', + measurementType: 'functional crown type (OL)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'fct_sf', + measurementType: 'functional crown type (SF)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'fct_ot', + measurementType: 'functional crown type (OT)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'fct_cm', + measurementType: 'functional crown type (CM)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, { field: 'mesowear', measurementType: 'mesowear', @@ -337,6 +577,55 @@ const MEASUREMENT_FIELD_MAPPINGS: Array<{ // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. measurementMethod: '', }, + { + field: 'mw_or_high', + measurementType: 'cusp relief high (OR%)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'mw_or_low', + measurementType: 'cusp relief low (OR%)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'mw_cs_sharp', + measurementType: 'cusp shape sharp (CS%)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'mw_cs_round', + measurementType: 'cusp shape round (CS%)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'mw_cs_blunt', + measurementType: 'cusp shape blunt (CS%)', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'mw_scale_min', + measurementType: 'mesowear scale min', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, + { + field: 'mw_scale_max', + measurementType: 'mesowear scale max', + measurementUnit: '', + // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. + measurementMethod: '', + }, { field: 'mw_value', measurementType: 'mesowear value', @@ -344,6 +633,20 @@ const MEASUREMENT_FIELD_MAPPINGS: Array<{ // TODO(#1150): No matching field description found on pantherion.com/dbmanual97/VSP.html. measurementMethod: '', }, + { + field: 'pop_struc', + measurementType: 'population structure', + measurementUnit: '', + measurementMethod: + 'Occasionally there will be evidence of herding or other gregarious behavior for a species. This could include evidence from mass deaths, well-preserved trace fossils (e.g., trackways), nesting-site or burrow aggregations, or association of individuals in burrows. It could also be based, less directly, on other aspects of the organism\'s biology -- for example, sexual dimorphism in sexual display or combat features. If so, indicate "soc" here and give details briefly in the Comment field. The choice "sol" (solitary) is allowed for completeness, but ordinarily there will be no positive evidence for solitary behavior, so the alternative to "soc" is usually a blank.', + }, + { + field: 'sp_status', + measurementType: 'species status', + measurementUnit: '', + // TODO(#1150): Add field description / meaning for NOW database usage. + measurementMethod: '', + }, ] export const mapSpeciesToMeasurementRows = (species: SpeciesForMeasurementExport): MeasurementCsvRow[] => { @@ -368,7 +671,6 @@ export const mapSpeciesToMeasurementRows = (species: SpeciesForMeasurementExport measurementValue, measurementUnit: mapping.measurementUnit, measurementMethod: mapping.measurementMethod, - measurementRemarks: '', }, ] }) @@ -410,7 +712,6 @@ const DWC_TERMS = { measurementValue: 'http://rs.tdwg.org/dwc/terms/measurementValue', measurementUnit: 'http://rs.tdwg.org/dwc/terms/measurementUnit', measurementMethod: 'http://rs.tdwg.org/dwc/terms/measurementMethod', - measurementRemarks: 'http://rs.tdwg.org/dwc/terms/measurementRemarks', }, } as const @@ -522,20 +823,58 @@ export const fetchSpeciesForDwcExport = async (): Promise< common_name: true, sp_author: true, sp_comment: true, + strain: true, + gene: true, + taxon_status: true, body_mass: true, brain_mass: true, + sv_length: true, + sd_size: true, + sd_display: true, + tshm: true, + symph_mob: true, + relative_blade_length: true, + tht: true, diet1: true, diet2: true, diet3: true, diet_description: true, + rel_fib: true, + selectivity: true, + digestion: true, + feedinghab1: true, + feedinghab2: true, + shelterhab1: true, + shelterhab2: true, locomo1: true, locomo2: true, locomo3: true, + hunt_forage: true, activity: true, crowntype: true, microwear: true, mesowear: true, + horizodonty: true, + cusp_shape: true, + cusp_count_buccal: true, + cusp_count_lingual: true, + loph_count_lon: true, + loph_count_trs: true, + fct_al: true, + fct_ol: true, + fct_sf: true, + fct_ot: true, + fct_cm: true, + mw_or_high: true, + mw_or_low: true, + mw_cs_sharp: true, + mw_cs_round: true, + mw_cs_blunt: true, + mw_scale_min: true, + mw_scale_max: true, mw_value: true, + pop_struc: true, + sp_status: true, }, }) } diff --git a/backend/src/unit-tests/dwcArchiveExport.test.ts b/backend/src/unit-tests/dwcArchiveExport.test.ts index 50df11ae..b12c8547 100644 --- a/backend/src/unit-tests/dwcArchiveExport.test.ts +++ b/backend/src/unit-tests/dwcArchiveExport.test.ts @@ -47,20 +47,58 @@ describe('DwC-A export mapping', () => { it('generates measurement rows only for meaningful values', () => { const rows = mapSpeciesToMeasurementRows({ species_id: 123, + strain: null, + gene: null, + taxon_status: null, body_mass: BigInt(2500), brain_mass: null, + sv_length: null, + sd_size: null, + sd_display: null, + tshm: null, + symph_mob: null, + relative_blade_length: null, + tht: null, diet1: '-', diet2: 'Herbivore', diet3: '', diet_description: 'Leaves', + rel_fib: null, + selectivity: null, + digestion: null, + feedinghab1: null, + feedinghab2: null, + shelterhab1: null, + shelterhab2: null, locomo1: null, locomo2: 'Arboreal', locomo3: null, + hunt_forage: null, activity: 'Diurnal', crowntype: null, microwear: 'High', + horizodonty: null, + cusp_shape: null, + cusp_count_buccal: null, + cusp_count_lingual: null, + loph_count_lon: null, + loph_count_trs: null, + fct_al: null, + fct_ol: null, + fct_sf: null, + fct_ot: null, + fct_cm: null, mesowear: null, + mw_or_high: null, + mw_or_low: null, + mw_cs_sharp: null, + mw_cs_round: null, + mw_cs_blunt: null, + mw_scale_min: null, + mw_scale_max: null, mw_value: 1.5, + pop_struc: null, + sp_status: null, }) const ids = rows.map(row => row.measurementID) @@ -93,20 +131,58 @@ describe('DwC-A export mapping', () => { common_name: 'Human', sp_author: null, sp_comment: null, + strain: null, + gene: null, + taxon_status: null, body_mass: BigInt(70000), brain_mass: 1350, + sv_length: null, + sd_size: null, + sd_display: null, + tshm: null, + symph_mob: null, + relative_blade_length: null, + tht: null, diet1: null, diet2: null, diet3: null, diet_description: null, + rel_fib: null, + selectivity: null, + digestion: null, + feedinghab1: null, + feedinghab2: null, + shelterhab1: null, + shelterhab2: null, locomo1: null, locomo2: null, locomo3: null, + hunt_forage: null, activity: null, crowntype: null, microwear: null, mesowear: null, + horizodonty: null, + cusp_shape: null, + cusp_count_buccal: null, + cusp_count_lingual: null, + loph_count_lon: null, + loph_count_trs: null, + fct_al: null, + fct_ol: null, + fct_sf: null, + fct_ot: null, + fct_cm: null, + mw_or_high: null, + mw_or_low: null, + mw_cs_sharp: null, + mw_cs_round: null, + mw_cs_blunt: null, + mw_scale_min: null, + mw_scale_max: null, mw_value: null, + pop_struc: null, + sp_status: null, }, ]) diff --git a/documentation/functionality/dwc_export.md b/documentation/functionality/dwc_export.md index f5fbb894..98a9b4de 100644 --- a/documentation/functionality/dwc_export.md +++ b/documentation/functionality/dwc_export.md @@ -70,24 +70,61 @@ Columns: - `measurementType` / `measurementUnit` / `measurementValue` per field mapping - `verbatimMeasurementType` = original DB field name (e.g. `diet1`, `body_mass`) - `measurementMethod` = Pantheria VSP field description where available (`https://www.pantherion.com/dbmanual97/VSP.html`) -- `measurementRemarks` = empty -v1 includes only these `com_species` fields (rows emitted only when source value is non-null and non-empty; `-` is treated as empty): +v1 includes these `com_species` fields (rows emitted only when source value is non-null and non-empty; `-` is treated as empty): +- `strain` +- `gene` +- `taxon_status` - `body_mass` → type: `body mass`, unit: `g` - `brain_mass` → type: `brain mass`, unit: `g` +- `sv_length` +- `sd_size` +- `sd_display` +- `tshm` +- `symph_mob` +- `relative_blade_length` +- `tht` - `diet1` → type: `diet category 1` - `diet2` → type: `diet category 2` - `diet3` → type: `diet category 3` - `diet_description` → type: `diet description` +- `rel_fib` +- `selectivity` +- `digestion` +- `feedinghab1` +- `feedinghab2` +- `shelterhab1` +- `shelterhab2` - `locomo1` → type: `locomotion 1` - `locomo2` → type: `locomotion 2` - `locomo3` → type: `locomotion 3` +- `hunt_forage` - `activity` → type: `activity` - `crowntype` → type: `crown type` - `microwear` → type: `microwear` - `mesowear` → type: `mesowear` +- `horizodonty` +- `cusp_shape` +- `cusp_count_buccal` +- `cusp_count_lingual` +- `loph_count_lon` +- `loph_count_trs` +- `fct_al` +- `fct_ol` +- `fct_sf` +- `fct_ot` +- `fct_cm` +- `mw_or_high` +- `mw_or_low` +- `mw_cs_sharp` +- `mw_cs_round` +- `mw_cs_blunt` +- `mw_scale_min` +- `mw_scale_max` - `mw_value` → type: `mesowear value` +- `pop_struc` +- `sp_status` ## Extension points (TODOs)