-
-
-
-
-
-
-
-
-
The following script contains basic steps for cleaning and filtering
-biodiversity records after the downloading process. It is organised by
-module as it is shown in OCCUR.
-
Important: These example filters GBIF records of three species of
-mosses from Spain and Portugal and keep only those that are placed in
-Portugal
-
-
-
-
Load packages
-
-
-
-
library(rgbif)
-library(bdc)
-library(tidyverse) # data manipulation
-library(dplyr)
-library(GADMTools) # administrative units and GIS
-library(CoordinateCleaner) # data cleaning functions
-library(countrycode) # country names standardization
-library(biogeo) # Environment and data
-
-
-
-
-
-
Prepare the environment
-
-
-
-
rm(list = ls(all.names = TRUE))
-
-# wd <- '' # Write working directory path
-# setwd(wd)
-
-
-
-
-
-
Get data
-
-
Only run section a if you want to download your own dataset (needs
-user account in GBIF)
-
-
-
a) Download the dataset from GBIF
-
-
-
-
spList <- c('Homalothecium aureum', 'Syntrichia ruralis', 'Tortella squarrosa')
-gbif_taxKeys <- name_backbone_checklist(spList) %>%
- filter(!matchType == 'NONE') %>%
- pull(usageKey)
-
-# occ_download(pred_in('taxonKey', gbif_taxKeys), # download 3 species names from our list
-# pred('hasCoordinate', TRUE), # this example only download records with associated coordinates
-# format = 'SIMPLE_CSV', user = 'xxx', pwd = 'xxx',email = 'xxx')
-# include user information
-
-# Do not run:
-# occ_download_wait('xxxxxxx-xxxxxxx')
-# occData <- occ_download_get('xxxxxxx-xxxxxxx') %>%
-# occ_download_import()
-
-
-
-
-
-
b) Load dataset: Start here using the data from GitHub
-
-
-
Read occurrences csv and check dimensions
-
-
-
-
data <- read.csv("recordsGBIFOCCUR.csv", encoding ='UTF-8', sep="\t")
-
-
-
-
-
-
Reduced version of the GBIF dataset
-
Filter fields
-
-
-
-
colnames(data)
-data <- data %>% select(c("gbifID", "family", "genus", "species", "infraspecificEpithet", "taxonRank", "scientificName", "countryCode", "locality", "stateProvince", "decimalLatitude", "decimalLongitude", "coordinatePrecision", "day", "month", "year", "occurrenceStatus", "basisOfRecord", "recordedBy", "issue"))
-colnames(data)
-
-
-
-
-
-
-
-
MODULE 1: Basis Of Record
-
What type of occurrences do we have?
-
-
-
-
unique(data$basisOfRecord)
-
-
-
-
Remove records without appropriate basis of record:
-
-
-
-
data <- data %>% filter(basisOfRecord == 'PRESERVED_SPECIMEN') # in this example we keep only preserved specimens
-
-
-
-
-
-
-
MODULE 2: Taxonomy
-
-
1. Is the record identified at a proper taxonomic rank?
-
Let’s see what do we have in our dataset:
-
-
-
-
unique(data$taxonRank)
-
-
-
-
Filter those records with appropriate taxonomic rank
-
-
-
-
data <- data %>% filter(taxonRank == 'SUBSPECIES' | taxonRank == 'SPECIES' )
-
-
-
-
-
-
-
-
MODULE 3: Geography
-
-
1. Check coordinates values
-
Discard records with latitude/longitude values equals to zero, exact
-same value or NULL
-
-
-
-
data <- data %>%
- filter(decimalLatitude != decimalLongitude) %>%
- filter(between(decimalLatitude, -90, 90)) %>%
- filter(between(decimalLongitude, -180, 180)) %>%
- filter(decimalLatitude != 0) %>%
- filter(decimalLongitude != 0)
-
-
-
-
-
-
2. Check coordinates precision
-
Number of decimal digits of coordinates as a measure of precision
-
Function to count number of decimals:
-
-
-
-
#Filter coordinates with at least 1 decimal place
-data <- bdc_coordinates_precision(data = data,
- lat ='decimalLatitude',
- lon = 'decimalLongitude', ndec = 1)
-data <- data %>% filter(.rou == 'TRUE')
-data$.rou <- NULL
-
-
-
-
-
-
3. Check records that don’t meet location criteria
-
Check if coordinates are placed in the assigned country, in the
-following example we check whether records are located in Portugal
-
-
Point in polygon by country analysis
-
Import a shapefile of the study area with Adm. Units borders at
-country level
-
Download world / country shapefile from https://gadm.org/download_world.html
-
-
-
-
countries <- read_sf('gadm41_PRT.gpkg') # Set the correct projection
-st_crs(countries) <- "+proj=longlat +datum=WGS84 +no_defs"
-
-
-
-
-
-
-
Point in polygon: join each point to a polygon based on
-position
-
-
-
-
data <- st_join(datapoints, countries) # Exclude those that didn't fall in the country polygon or in 'SEA'
-data <- data %>% filter(COUNTRY == "Portugal")
-# Translate 'countryCode' information (ISO 3166-1-alpha-2 = "iso2c") into country names of GADM 'countryCode'
-data$countryName <- countrycode(data$countryCode,
- origin = "iso2c",
- destination = "country.name")
-# Check the names obtained
-unique(data$countryName)
-# Match country names and label as 'FALSE' errors of location
-data <- data %>%
- mutate(countryCheck = case_when(COUNTRY != data$countryName ~ FALSE,
- TRUE ~ TRUE))
-# Subset and extract records located in country assigned by collector ('correct')
-data <- data %>% filter(countryCheck == 'TRUE')
-data <- as.data.frame(data)
-
-
-
-
-
-
Delete Centroids:
-
-
-
-
# Label coordinates placed in centroids of the country or capital
-cap <- cc_cap (data, lon = "decimalLongitude",
- lat = "decimalLatitude", value = "flagged")
-data <- cbind(data, cap)
-cen <- cc_cen (data, lon = "decimalLongitude",
- lat = "decimalLatitude", value = "flagged")
-data <- cbind(data, cen)
-# Label coordinates placed in gbif headquarters
-gbif <- cc_gbif(data, lon = "decimalLongitude",
- lat = "decimalLatitude", value = "flagged")
-data <- cbind(data, gbif)
-# Label coordinates from museums, gardens, institutions, zoo's…
-inst <- cc_inst(data, lon = "decimalLongitude",
- lat = "decimalLatitude", value = "flagged")
-data <- cbind(data, inst)
-#Filter and exclude centroids:
-data <- data %>% filter(cap == 'TRUE') %>%
- filter(cen == 'TRUE') %>%
- filter(gbif == 'TRUE') %>%
- filter(inst == 'TRUE')
-
-
-
-
-
-
-
-
-
MODULE 4: Time
-
-
-
b. Set temporal range of time:
-
-
-
-
data <- data %>% filter(year >= 1980 & year <= 2020)
-
-
-
-
-
-
-
-
MODULE 5: Duplicates
-
Here we have multiple combination to detect duplicate records .
-
Choose the appropriate combination of fields based on your study:
-
-
SpeciesName + Lat Lon + Date + recordedBy
-
-
-
-
dataDuplic1 <- cc_dupl (data, lon = "decimalLongitude",
- lat = "decimalLatitude",
- species = "species",
- additions = c('recordedBy', 'year'))
-
-
-
-
-
-
SpeciesName + Lat Lon + recordedBy
-
-
-
-
dataDuplic2 <- cc_dupl (data, lon = "decimalLongitude",
- lat = "decimalLatitude",
- species = "species",
- additions = 'recordedBy')
-
-
-
-
-
-
SpeciesName + Lat Lon + Year
-
-
-
-
dataDuplic3 <- cc_dupl (data, lon = "decimalLongitude",
- lat = "decimalLatitude",
- species = "species",
- additions = 'year')
-
-
-
-
-
-
SpeciesName + Lat Lon
-
-
-
-
dataDuplic4 <- cc_dupl (data, lon = "decimalLongitude",
- lat = "decimalLatitude",
- species = "species")
-
-
-
-
-
-
SpeciesName + Lat Lon with buffer or rounded coordinates
-
-
-
-
data$lon_round <- round(data$decimalLongitude, 2)
-data$lat_round <- round(data$decimalLatitude, 2)
-dataDuplic5 <- cc_dupl (data, lon = "lon_round",
- lat = "lat_round", species = "species")
-
-
-
-
-
-
SpeciesName + cell
-
-
-
-
datbiogeo_b <- keepmainfields(data,
- ID='gbifID',
- Species='species',
- x='decimalLongitude', y='decimalLatitude')
-dataDuplic6 <- duplicatesexclude(datbiogeo_b, 10) # spatial resolution in minutes
-dataDuplic6 <- merge(data, dataDuplic6, by.x='gbifID', by.y='ID', all.x=TRUE)
-dataDuplic6 <- dataDuplic6 %>% filter(Exclude == 0)
-
-
-
-
-
-
-
SAVE THE CLEANED DATASET
-
-
-
Here SpeciesName + Lat Lon with rounded coordinates
-
-
-
-
write.table(dataDuplic5, 'final_data.csv', sep="\t", row.names = FALSE)
-
-
-
-
-
-
LS0tDQp0aXRsZTogIkV4YW1wbGVDYXNlX1JDb2RlIg0KYXV0aG9yOiAiQ1JGIg0KZGF0ZTogIjIwMjMtMTEtMjQiDQpvdXRwdXQ6DQogIGh0bWxfZG9jdW1lbnQ6DQogICAgZGZfcHJpbnQ6IHBhZ2VkDQogIGh0bWxfbm90ZWJvb2s6IGRlZmF1bHQNCi0tLQ0KDQpUaGUgZm9sbG93aW5nIHNjcmlwdCBjb250YWlucyBiYXNpYyBzdGVwcyBmb3IgY2xlYW5pbmcgYW5kIGZpbHRlcmluZyBiaW9kaXZlcnNpdHkgcmVjb3JkcyBhZnRlciB0aGUgZG93bmxvYWRpbmcgcHJvY2Vzcy4gSXQgaXMgb3JnYW5pc2VkIGJ5IG1vZHVsZSBhcyBpdCBpcyBzaG93biBpbiBPQ0NVUi4NCg0KSW1wb3J0YW50OiBUaGVzZSBleGFtcGxlIGZpbHRlcnMgR0JJRiByZWNvcmRzIG9mIHRocmVlIHNwZWNpZXMgb2YgbW9zc2VzIGZyb20gU3BhaW4gYW5kIFBvcnR1Z2FsIGFuZCBrZWVwIG9ubHkgdGhvc2UgdGhhdCBhcmUgcGxhY2VkIGluIFBvcnR1Z2FsDQoNCmBgYHtyIGtsaXBweSwgaW5jbHVkZT1GQUxTRX0NCmtsaXBweTo6a2xpcHB5KHBvc2l0aW9uID0gYygndG9wJywgJ3JpZ2h0JyksIGNvbG9yID0gJ2RhcmtyZWQnKQ0KYGBgDQoNCiMjIyBMb2FkIHBhY2thZ2VzDQoNCmBgYHtyIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0UsIGNsYXNzLnNvdXJjZT0na2xpcHB5J30NCmxpYnJhcnkocmdiaWYpIA0KbGlicmFyeShiZGMpIA0KbGlicmFyeSh0aWR5dmVyc2UpICMgZGF0YSBtYW5pcHVsYXRpb24gDQpsaWJyYXJ5KGRwbHlyKSANCmxpYnJhcnkoR0FETVRvb2xzKSAjIGFkbWluaXN0cmF0aXZlIHVuaXRzIGFuZCBHSVMgDQpsaWJyYXJ5KENvb3JkaW5hdGVDbGVhbmVyKSAjIGRhdGEgY2xlYW5pbmcgZnVuY3Rpb25zIA0KbGlicmFyeShjb3VudHJ5Y29kZSkgIyBjb3VudHJ5IG5hbWVzIHN0YW5kYXJkaXphdGlvbiANCmxpYnJhcnkoYmlvZ2VvKSAjIEVudmlyb25tZW50IGFuZCBkYXRhIA0KYGBgDQoNCiMjIyBQcmVwYXJlIHRoZSBlbnZpcm9ubWVudA0KDQpgYGB7ciBjbGFzcy5zb3VyY2U9J2tsaXBweSd9DQpybShsaXN0ID0gbHMoYWxsLm5hbWVzID0gVFJVRSkpIA0KDQojIHdkIDwtICcnICMgV3JpdGUgd29ya2luZyBkaXJlY3RvcnkgcGF0aCANCiMgc2V0d2Qod2QpDQpgYGANCg0KIyMjIEdldCBkYXRhDQoNCiMjIyMgT25seSBydW4gc2VjdGlvbiBhIGlmIHlvdSB3YW50IHRvIGRvd25sb2FkIHlvdXIgb3duIGRhdGFzZXQgKG5lZWRzIHVzZXIgYWNjb3VudCBpbiBHQklGKQ0KDQojIyMjIGEpIERvd25sb2FkIHRoZSBkYXRhc2V0IGZyb20gR0JJRg0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0Kc3BMaXN0IDwtIGMoJ0hvbWFsb3RoZWNpdW0gYXVyZXVtJywgJ1N5bnRyaWNoaWEgcnVyYWxpcycsICdUb3J0ZWxsYSBzcXVhcnJvc2EnKQ0KZ2JpZl90YXhLZXlzIDwtIG5hbWVfYmFja2JvbmVfY2hlY2tsaXN0KHNwTGlzdCkgJT4lIA0KICBmaWx0ZXIoIW1hdGNoVHlwZSA9PSAnTk9ORScpICU+JSANCiAgcHVsbCh1c2FnZUtleSkNCg0KIyBvY2NfZG93bmxvYWQocHJlZF9pbigndGF4b25LZXknLCBnYmlmX3RheEtleXMpLCAjIGRvd25sb2FkIDMgc3BlY2llcyBuYW1lcyBmcm9tIG91ciBsaXN0IA0KIyBwcmVkKCdoYXNDb29yZGluYXRlJywgVFJVRSksICMgdGhpcyBleGFtcGxlIG9ubHkgZG93bmxvYWQgcmVjb3JkcyB3aXRoIGFzc29jaWF0ZWQgY29vcmRpbmF0ZXMgDQojIGZvcm1hdCA9ICdTSU1QTEVfQ1NWJywgdXNlciA9ICd4eHgnLCBwd2QgPSAneHh4JyxlbWFpbCA9ICd4eHgnKSANCiMgaW5jbHVkZSB1c2VyIGluZm9ybWF0aW9uDQoNCiMgRG8gbm90IHJ1bjogDQojIG9jY19kb3dubG9hZF93YWl0KCd4eHh4eHh4LXh4eHh4eHgnKSANCiMgb2NjRGF0YSA8LSBvY2NfZG93bmxvYWRfZ2V0KCd4eHh4eHh4LXh4eHh4eHgnKSAlPiUgDQojICAgICAgICAgICAgIG9jY19kb3dubG9hZF9pbXBvcnQoKQ0KYGBgDQoNCiMjIyMgYikgTG9hZCBkYXRhc2V0OiBTdGFydCBoZXJlIHVzaW5nIHRoZSBkYXRhIGZyb20gR2l0SHViDQoNCiMjIyMgUmVhZCBvY2N1cnJlbmNlcyBjc3YgYW5kIGNoZWNrIGRpbWVuc2lvbnMNCg0KYGBge3IsIGNsYXNzLnNvdXJjZT0na2xpcHB5J30NCmRhdGEgPC0gcmVhZC5jc3YoInJlY29yZHNHQklGT0NDVVIuY3N2IiwgZW5jb2RpbmcgPSdVVEYtOCcsIHNlcD0iXHQiKQ0KYGBgDQoNCiMjIyMgUmVkdWNlZCB2ZXJzaW9uIG9mIHRoZSBHQklGIGRhdGFzZXQNCg0KRmlsdGVyIGZpZWxkcw0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0KY29sbmFtZXMoZGF0YSkgDQpkYXRhIDwtIGRhdGEgJT4lIHNlbGVjdChjKCJnYmlmSUQiLCAiZmFtaWx5IiwgImdlbnVzIiwgInNwZWNpZXMiLCAiaW5mcmFzcGVjaWZpY0VwaXRoZXQiLCAidGF4b25SYW5rIiwgInNjaWVudGlmaWNOYW1lIiwgImNvdW50cnlDb2RlIiwgImxvY2FsaXR5IiwgInN0YXRlUHJvdmluY2UiLCAiZGVjaW1hbExhdGl0dWRlIiwgImRlY2ltYWxMb25naXR1ZGUiLCAiY29vcmRpbmF0ZVByZWNpc2lvbiIsICJkYXkiLCAibW9udGgiLCAieWVhciIsICJvY2N1cnJlbmNlU3RhdHVzIiwgImJhc2lzT2ZSZWNvcmQiLCAicmVjb3JkZWRCeSIsICJpc3N1ZSIpKSANCmNvbG5hbWVzKGRhdGEpDQpgYGANCg0KLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tDQoNCiMjIE1PRFVMRSAxOiBCYXNpcyBPZiBSZWNvcmQNCg0KV2hhdCB0eXBlIG9mIG9jY3VycmVuY2VzIGRvIHdlIGhhdmU/DQoNCmBgYHtyLCBjbGFzcy5zb3VyY2U9J2tsaXBweSd9DQp1bmlxdWUoZGF0YSRiYXNpc09mUmVjb3JkKSANCmBgYA0KDQpSZW1vdmUgcmVjb3JkcyB3aXRob3V0IGFwcHJvcHJpYXRlIGJhc2lzIG9mIHJlY29yZDoNCg0KYGBge3IsIGNsYXNzLnNvdXJjZT0na2xpcHB5J30NCmRhdGEgPC0gZGF0YSAlPiUgZmlsdGVyKGJhc2lzT2ZSZWNvcmQgPT0gJ1BSRVNFUlZFRF9TUEVDSU1FTicpICMgaW4gdGhpcyBleGFtcGxlIHdlIGtlZXAgb25seSBwcmVzZXJ2ZWQgc3BlY2ltZW5zDQpgYGANCg0KLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tDQoNCiMjIE1PRFVMRSAyOiBUYXhvbm9teQ0KDQojIyMgMS4gSXMgdGhlIHJlY29yZCBpZGVudGlmaWVkIGF0IGEgcHJvcGVyIHRheG9ub21pYyByYW5rPw0KDQpMZXQncyBzZWUgd2hhdCBkbyB3ZSBoYXZlIGluIG91ciBkYXRhc2V0Og0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0KdW5pcXVlKGRhdGEkdGF4b25SYW5rKQ0KYGBgDQoNCkZpbHRlciB0aG9zZSByZWNvcmRzIHdpdGggYXBwcm9wcmlhdGUgdGF4b25vbWljIHJhbmsNCg0KYGBge3IsIGNsYXNzLnNvdXJjZT0na2xpcHB5J30NCmRhdGEgPC0gZGF0YSAlPiUgZmlsdGVyKHRheG9uUmFuayA9PSAnU1VCU1BFQ0lFUycgfCB0YXhvblJhbmsgPT0gJ1NQRUNJRVMnICkNCmBgYA0KDQojIyMgMi4gU2VsZWN0IG9ubHkgcmVjb3JkcyB0aGF0IGluY2x1ZGUgYXV0aG9yc2hpcCBpbmZvcm1hdGlvbiBpbiB0aGVpciBvcmlnaW5hbCBzY2llbnRpZmljIG5hbWVzIHByb3Bvc2VkDQoNCkV4dHJhY3QgYXV0aG9ycyBuYW1lIGluY2x1ZGVkIGluIHNjaWVudGlmTmFtZSBpbmZvcm1hdGlvbg0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0KZGF0YSRhdXRob3JzaGlwTmFtZSA8LSB3b3JkKGRhdGEkc2NpZW50aWZpY05hbWUsIDMsIC0xKQ0KYGBgDQoNCkRlbGV0ZSByZWNvcmRzIHdpdGggbm8gbmFtZSBpbmNsdWRlZA0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0KZGF0YSA8LSBkYXRhICU+JSBmaWx0ZXIoIWlzLm5hKGF1dGhvcnNoaXBOYW1lKSkNCmBgYA0KDQotLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0NCg0KIyMgTU9EVUxFIDM6IEdlb2dyYXBoeQ0KDQojIyMgMS4gQ2hlY2sgY29vcmRpbmF0ZXMgdmFsdWVzDQoNCkRpc2NhcmQgcmVjb3JkcyB3aXRoIGxhdGl0dWRlL2xvbmdpdHVkZSB2YWx1ZXMgZXF1YWxzIHRvIHplcm8sIGV4YWN0IHNhbWUgdmFsdWUgb3IgTlVMTA0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0KZGF0YSA8LSBkYXRhICU+JSANCiAgICAgICAgICBmaWx0ZXIoZGVjaW1hbExhdGl0dWRlICE9IGRlY2ltYWxMb25naXR1ZGUpICU+JSANCiAgICAgICAgICBmaWx0ZXIoYmV0d2VlbihkZWNpbWFsTGF0aXR1ZGUsIC05MCwgOTApKSAlPiUgDQogICAgICAgICAgZmlsdGVyKGJldHdlZW4oZGVjaW1hbExvbmdpdHVkZSwgLTE4MCwgMTgwKSkgJT4lIA0KICAgICAgICAgIGZpbHRlcihkZWNpbWFsTGF0aXR1ZGUgIT0gMCkgJT4lIA0KICAgICAgICAgIGZpbHRlcihkZWNpbWFsTG9uZ2l0dWRlICE9IDApDQpgYGANCg0KIyMjIDIuIENoZWNrIGNvb3JkaW5hdGVzIHByZWNpc2lvbg0KDQpOdW1iZXIgb2YgZGVjaW1hbCBkaWdpdHMgb2YgY29vcmRpbmF0ZXMgYXMgYSBtZWFzdXJlIG9mIHByZWNpc2lvbg0KDQpGdW5jdGlvbiB0byBjb3VudCBudW1iZXIgb2YgZGVjaW1hbHM6DQoNCmBgYHtyLCBjbGFzcy5zb3VyY2U9J2tsaXBweSd9DQojRmlsdGVyIGNvb3JkaW5hdGVzIHdpdGggYXQgbGVhc3QgMSBkZWNpbWFsIHBsYWNlDQpkYXRhIDwtIGJkY19jb29yZGluYXRlc19wcmVjaXNpb24oZGF0YSA9IGRhdGEsIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxhdCA9J2RlY2ltYWxMYXRpdHVkZScsIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxvbiA9ICdkZWNpbWFsTG9uZ2l0dWRlJywgbmRlYyA9IDEpIA0KZGF0YSA8LSBkYXRhICU+JSBmaWx0ZXIoLnJvdSA9PSAnVFJVRScpIA0KZGF0YSQucm91IDwtIE5VTEwgDQpgYGANCg0KIyMjIDMuIENoZWNrIHJlY29yZHMgdGhhdCBkb24ndCBtZWV0IGxvY2F0aW9uIGNyaXRlcmlhDQoNCkNoZWNrIGlmIGNvb3JkaW5hdGVzIGFyZSBwbGFjZWQgaW4gdGhlIGFzc2lnbmVkIGNvdW50cnksIGluIHRoZSBmb2xsb3dpbmcgZXhhbXBsZSB3ZSBjaGVjayB3aGV0aGVyIHJlY29yZHMgYXJlIGxvY2F0ZWQgaW4gUG9ydHVnYWwNCg0KIyMjIyBQb2ludCBpbiBwb2x5Z29uIGJ5IGNvdW50cnkgYW5hbHlzaXMNCg0KSW1wb3J0IGEgc2hhcGVmaWxlIG9mIHRoZSBzdHVkeSBhcmVhIHdpdGggQWRtLiBVbml0cyBib3JkZXJzIGF0IGNvdW50cnkgbGV2ZWwNCg0KRG93bmxvYWQgd29ybGQgLyBjb3VudHJ5IHNoYXBlZmlsZSBmcm9tIDxodHRwczovL2dhZG0ub3JnL2Rvd25sb2FkX3dvcmxkLmh0bWw+DQoNCmBgYHtyIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0UsIGNsYXNzLnNvdXJjZT0na2xpcHB5J30NCmNvdW50cmllcyA8LSByZWFkX3NmKCdnYWRtNDFfUFJULmdwa2cnKSAjIFNldCB0aGUgY29ycmVjdCBwcm9qZWN0aW9uIA0Kc3RfY3JzKGNvdW50cmllcykgPC0gIitwcm9qPWxvbmdsYXQgK2RhdHVtPVdHUzg0ICtub19kZWZzIg0KYGBgDQoNCiMjIyMgVHJhbnNmb3JtIG9jY3VycmVuY2VzIGludG8gc3BhdGlhbCBwb2ludHMgYW5kIHByb2plY3Q6DQoNCmBgYHtyLCBjbGFzcy5zb3VyY2U9J2tsaXBweSd9DQpkYXRhJHggPC0gZGF0YSRkZWNpbWFsTG9uZ2l0dWRlIA0KZGF0YSR5IDwtIGRhdGEkZGVjaW1hbExhdGl0dWRlDQoNCmRhdGFwb2ludHMgPC0gc3RfYXNfc2YoeCA9IGRhdGEsY29vcmRzID0gYygieCIsICJ5IiksIA0KICAgICAgICAgICAgICAgICAgICAgICBjcnMgPSAiK3Byb2o9bG9uZ2xhdCArZGF0dW09V0dTODQgK25vX2RlZnMiKQ0KYGBgDQoNCiMjIyMgUG9pbnQgaW4gcG9seWdvbjogam9pbiBlYWNoIHBvaW50IHRvIGEgcG9seWdvbiBiYXNlZCBvbiBwb3NpdGlvbg0KDQpgYGB7ciBldmFsPUZBTFNFLCBjbGFzcy5zb3VyY2U9J2tsaXBweSd9DQpkYXRhIDwtIHN0X2pvaW4oZGF0YXBvaW50cywgY291bnRyaWVzKSAjIEV4Y2x1ZGUgdGhvc2UgdGhhdCBkaWRuJ3QgZmFsbCBpbiB0aGUgY291bnRyeSBwb2x5Z29uIG9yIGluICdTRUEnIA0KZGF0YSA8LSBkYXRhICU+JSBmaWx0ZXIoQ09VTlRSWSA9PSAiUG9ydHVnYWwiKSANCiMgVHJhbnNsYXRlICdjb3VudHJ5Q29kZScgaW5mb3JtYXRpb24gKElTTyAzMTY2LTEtYWxwaGEtMiA9ICJpc28yYyIpIGludG8gY291bnRyeSBuYW1lcyBvZiBHQURNICdjb3VudHJ5Q29kZScgDQpkYXRhJGNvdW50cnlOYW1lIDwtIGNvdW50cnljb2RlKGRhdGEkY291bnRyeUNvZGUsIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvcmlnaW4gPSAiaXNvMmMiLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGVzdGluYXRpb24gPSAiY291bnRyeS5uYW1lIikNCiMgQ2hlY2sgdGhlIG5hbWVzIG9idGFpbmVkDQp1bmlxdWUoZGF0YSRjb3VudHJ5TmFtZSkNCiMgTWF0Y2ggY291bnRyeSBuYW1lcyBhbmQgbGFiZWwgYXMgJ0ZBTFNFJyBlcnJvcnMgb2YgbG9jYXRpb24NCmRhdGEgPC0gZGF0YSAlPiUgDQogIG11dGF0ZShjb3VudHJ5Q2hlY2sgPSBjYXNlX3doZW4oQ09VTlRSWSAhPSBkYXRhJGNvdW50cnlOYW1lIH4gRkFMU0UsIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFRSVUUgfiBUUlVFKSkgDQojIFN1YnNldCBhbmQgZXh0cmFjdCByZWNvcmRzIGxvY2F0ZWQgaW4gY291bnRyeSBhc3NpZ25lZCBieSBjb2xsZWN0b3IgKCdjb3JyZWN0JykgDQpkYXRhIDwtIGRhdGEgJT4lIGZpbHRlcihjb3VudHJ5Q2hlY2sgPT0gJ1RSVUUnKSANCmRhdGEgPC0gYXMuZGF0YS5mcmFtZShkYXRhKQ0KYGBgDQoNCiMjIyMgRGVsZXRlIENlbnRyb2lkczoNCg0KYGBge3IgZXZhbD1GQUxTRSwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0KIyBMYWJlbCBjb29yZGluYXRlcyBwbGFjZWQgaW4gY2VudHJvaWRzIG9mIHRoZSBjb3VudHJ5IG9yIGNhcGl0YWwgDQpjYXAgPC0gY2NfY2FwIChkYXRhLCBsb24gPSAiZGVjaW1hbExvbmdpdHVkZSIsIA0KICAgICAgICAgICAgICAgICAgICAgbGF0ID0gImRlY2ltYWxMYXRpdHVkZSIsIHZhbHVlID0gImZsYWdnZWQiKSANCmRhdGEgPC0gY2JpbmQoZGF0YSwgY2FwKSANCmNlbiA8LSBjY19jZW4gKGRhdGEsIGxvbiA9ICJkZWNpbWFsTG9uZ2l0dWRlIiwgDQogICAgICAgICAgICAgICAgICAgICBsYXQgPSAiZGVjaW1hbExhdGl0dWRlIiwgdmFsdWUgPSAiZmxhZ2dlZCIpIA0KZGF0YSA8LSBjYmluZChkYXRhLCBjZW4pICANCiMgTGFiZWwgY29vcmRpbmF0ZXMgcGxhY2VkIGluIGdiaWYgaGVhZHF1YXJ0ZXJzDQpnYmlmIDwtIGNjX2diaWYoZGF0YSwgbG9uID0gImRlY2ltYWxMb25naXR1ZGUiLCANCiAgICAgICAgICAgICAgICAgICAgbGF0ID0gImRlY2ltYWxMYXRpdHVkZSIsIHZhbHVlID0gImZsYWdnZWQiKSANCmRhdGEgPC0gY2JpbmQoZGF0YSwgZ2JpZikNCiMgTGFiZWwgY29vcmRpbmF0ZXMgZnJvbSBtdXNldW1zLCBnYXJkZW5zLCBpbnN0aXR1dGlvbnMsIHpvbydz4oCmDQppbnN0IDwtIGNjX2luc3QoZGF0YSwgbG9uID0gImRlY2ltYWxMb25naXR1ZGUiLA0KICAgICAgICAgICAgICAgICAgICAgIGxhdCA9ICJkZWNpbWFsTGF0aXR1ZGUiLCB2YWx1ZSA9ICJmbGFnZ2VkIikNCmRhdGEgPC0gY2JpbmQoZGF0YSwgaW5zdCkNCiNGaWx0ZXIgYW5kIGV4Y2x1ZGUgY2VudHJvaWRzOg0KZGF0YSA8LSBkYXRhICU+JSBmaWx0ZXIoY2FwID09ICdUUlVFJykgJT4lIA0KICAgICAgICAgICAgICAgICAgZmlsdGVyKGNlbiA9PSAnVFJVRScpICU+JSANCiAgICAgICAgICAgICAgICAgIGZpbHRlcihnYmlmID09ICdUUlVFJykgJT4lIA0KICAgICAgICAgICAgICAgICAgZmlsdGVyKGluc3QgPT0gJ1RSVUUnKQ0KICAgICAgICAgICAgICAgICAgDQpgYGANCg0KLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tDQoNCiMjIE1PRFVMRSA0OiBUaW1lDQoNCiMjIyMgYS4gQ2hvb3NlIGxldmVsIG9mIHRlbXBvcmFsIGluZm9ybWF0aW9uOg0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0KIyBPbmx5IHJlY29yZHMgd2l0aCBjb21wbGV0ZSBkYXRlIG9mIGNvbGxlY3Rpb24NCmRhdGEgPC0gZGF0YSAlPiUgDQogICAgICAgIGZpbHRlcighaXMubmEoeWVhcikgfCAhaXMubmEobW9udGgpIHwgIWlzLm5hKGRheSkpICANCiMgQ3JlYXRlIGEgZmllbGQgdGhhdCBjb21iaW5lcyBkYXkvbW9udGgveWVhciBpbmZvcm1hdGlvbg0KZGF0YSRkYXRlIDwtIHBhc3RlKGRhdGEkZGF5LCBkYXRhJG1vbnRoLCBkYXRhJHllYXIsIHNlcCA9ICcvJykNCmBgYA0KDQojIyMjIGIuIFNldCB0ZW1wb3JhbCByYW5nZSBvZiB0aW1lOg0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0KZGF0YSA8LSBkYXRhICU+JSBmaWx0ZXIoeWVhciA+PSAxOTgwICYgeWVhciA8PSAyMDIwKQ0KYGBgDQoNCi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQ0KDQojIyBNT0RVTEUgNTogRHVwbGljYXRlcw0KDQpIZXJlIHdlIGhhdmUgbXVsdGlwbGUgY29tYmluYXRpb24gdG8gZGV0ZWN0IGR1cGxpY2F0ZSByZWNvcmRzIC4NCg0KQ2hvb3NlIHRoZSBhcHByb3ByaWF0ZSBjb21iaW5hdGlvbiBvZiBmaWVsZHMgYmFzZWQgb24geW91ciBzdHVkeToNCg0KIyMjIyBTcGVjaWVzTmFtZSArIExhdCBMb24gKyBEYXRlICsgcmVjb3JkZWRCeQ0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0KZGF0YUR1cGxpYzEgPC0gY2NfZHVwbCAoZGF0YSwgbG9uID0gImRlY2ltYWxMb25naXR1ZGUiLCANCiAgICAgICAgICAgICAgICAgICAgICAgIGxhdCA9ICJkZWNpbWFsTGF0aXR1ZGUiLCANCiAgICAgICAgICAgICAgICAgICAgICAgIHNwZWNpZXMgPSAic3BlY2llcyIsIA0KICAgICAgICAgICAgICAgICAgICAgICAgYWRkaXRpb25zID0gYygncmVjb3JkZWRCeScsICd5ZWFyJykpDQpgYGANCg0KIyMjIyBTcGVjaWVzTmFtZSArIExhdCBMb24gKyByZWNvcmRlZEJ5DQoNCmBgYHtyLCBjbGFzcy5zb3VyY2U9J2tsaXBweSd9DQpkYXRhRHVwbGljMiA8LSBjY19kdXBsIChkYXRhLCBsb24gPSAiZGVjaW1hbExvbmdpdHVkZSIsIA0KICAgICAgICAgICAgICAgICAgICAgICAgbGF0ID0gImRlY2ltYWxMYXRpdHVkZSIsIA0KICAgICAgICAgICAgICAgICAgICAgICAgc3BlY2llcyA9ICJzcGVjaWVzIiwgDQogICAgICAgICAgICAgICAgICAgICAgICBhZGRpdGlvbnMgPSAncmVjb3JkZWRCeScpDQpgYGANCg0KIyMjIyBTcGVjaWVzTmFtZSArIExhdCBMb24gKyBZZWFyDQoNCmBgYHtyLCBjbGFzcy5zb3VyY2U9J2tsaXBweSd9DQpkYXRhRHVwbGljMyA8LSBjY19kdXBsIChkYXRhLCBsb24gPSAiZGVjaW1hbExvbmdpdHVkZSIsIA0KICAgICAgICAgICAgICAgICAgICAgICAgbGF0ID0gImRlY2ltYWxMYXRpdHVkZSIsIA0KICAgICAgICAgICAgICAgICAgICAgICAgc3BlY2llcyA9ICJzcGVjaWVzIiwgDQogICAgICAgICAgICAgICAgICAgICAgICBhZGRpdGlvbnMgPSAneWVhcicpDQpgYGANCg0KIyMjIyBTcGVjaWVzTmFtZSArIExhdCBMb24NCg0KYGBge3IsIGNsYXNzLnNvdXJjZT0na2xpcHB5J30NCmRhdGFEdXBsaWM0IDwtIGNjX2R1cGwgKGRhdGEsIGxvbiA9ICJkZWNpbWFsTG9uZ2l0dWRlIiwgDQogICAgICAgICAgICAgICAgICAgICAgICBsYXQgPSAiZGVjaW1hbExhdGl0dWRlIiwgDQogICAgICAgICAgICAgICAgICAgICAgICBzcGVjaWVzID0gInNwZWNpZXMiKQ0KYGBgDQoNCiMjIyMgU3BlY2llc05hbWUgKyBMYXQgTG9uIHdpdGggYnVmZmVyIG9yIHJvdW5kZWQgY29vcmRpbmF0ZXMNCg0KYGBge3IsIGNsYXNzLnNvdXJjZT0na2xpcHB5J30NCmRhdGEkbG9uX3JvdW5kIDwtIHJvdW5kKGRhdGEkZGVjaW1hbExvbmdpdHVkZSwgMikgDQpkYXRhJGxhdF9yb3VuZCA8LSByb3VuZChkYXRhJGRlY2ltYWxMYXRpdHVkZSwgMikgDQpkYXRhRHVwbGljNSA8LSBjY19kdXBsIChkYXRhLCBsb24gPSAibG9uX3JvdW5kIiwgDQogICAgICAgICAgICAgICAgICAgICAgICBsYXQgPSAibGF0X3JvdW5kIiwgc3BlY2llcyA9ICJzcGVjaWVzIikNCmBgYA0KDQojIyMjIFNwZWNpZXNOYW1lICsgY2VsbA0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0KZGF0YmlvZ2VvX2IgPC0ga2VlcG1haW5maWVsZHMoZGF0YSwgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJRD0nZ2JpZklEJywNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFNwZWNpZXM9J3NwZWNpZXMnLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgeD0nZGVjaW1hbExvbmdpdHVkZScsIHk9J2RlY2ltYWxMYXRpdHVkZScpIA0KZGF0YUR1cGxpYzYgPC0gZHVwbGljYXRlc2V4Y2x1ZGUoZGF0YmlvZ2VvX2IsIDEwKSAjIHNwYXRpYWwgcmVzb2x1dGlvbiBpbiBtaW51dGVzDQpkYXRhRHVwbGljNiA8LSBtZXJnZShkYXRhLCBkYXRhRHVwbGljNiwgYnkueD0nZ2JpZklEJywgYnkueT0nSUQnLCBhbGwueD1UUlVFKSANCmRhdGFEdXBsaWM2IDwtIGRhdGFEdXBsaWM2ICU+JSBmaWx0ZXIoRXhjbHVkZSA9PSAwKQ0KYGBgDQoNCi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQ0KDQojIyMjIFNBVkUgVEhFIENMRUFORUQgREFUQVNFVA0KDQojIyMjIEhlcmUgU3BlY2llc05hbWUgKyBMYXQgTG9uIHdpdGggcm91bmRlZCBjb29yZGluYXRlcw0KDQpgYGB7ciwgY2xhc3Muc291cmNlPSdrbGlwcHknfQ0Kd3JpdGUudGFibGUoZGF0YUR1cGxpYzUsICdmaW5hbF9kYXRhLmNzdicsIHNlcD0iXHQiLCByb3cubmFtZXMgPSBGQUxTRSkNCmBgYA0K
-
-
-
-