get_projects

#!/usr/bin/env node
const path = require('path')
const fetch = require('isomorphic-unfetch')
const yaml = require('yaml')
const fs = require('fs')
const imageThumbnail = require('image-thumbnail')

;(async () => {
  const csv = await fs.promises.readFile('projects_okhs.csv', 'utf-8')
  let projects = await Promise.all(
    csv
      .split('\n')
      .slice(1)
      .map(async (line, index) => {
        const [name, date, link] = line.split(',')
        if (link) {
          return fetchText(link)
            .then(text => {
              const origin = path.dirname(link) + '/'
              return { id: index, origin, ...yaml.parse(text) }
            })
            .catch(e => {
              console.warn(e)
              console.warn('--------------------------------------------')
              console.warn('Error reading:', link)
              console.warn('--------------------------------------------')
            })
        }
      }),
  )
  // remove null/undefined
  projects = projects.filter(x => x)
  projects = projects.map(processUrls)
  // remove null/undefined
  projects = projects.filter(x => x)
  shuffleArray(projects)
  projects = await Promise.all(projects.map(processImages))
  console.log(JSON.stringify(projects, null, 2))
})()

async function fetchText(url) {
  // just checking it's a valid url
  new URL(url)
  // actually fetch it
  return fetch(url).then(r => {
    if (r.status !== 200) {
      throw Error(r.status)
    }
    return r.text()
  })
}

function processUrls(project) {
  const origin = project.origin
  const docHome = project['documentation-home'] || project['project-link']
  if (docHome == null) {
    return
  }
  project['documentation-home'] = new URL(docHome, origin).href
  let image = project.image
  if (image) {
    project.image = new URL(image, origin).href
  }
  return project
}

async function processImages(project) {
  let image = project.image
  if (image) {
    const r = await fetch(image)
    if (r.status != 200 || !/^image\//.test(r.headers.get('Content-Type'))) {
      console.warn("Can't read image:", image)
      delete project.image
    } else {
      try {
        const ext = r.headers.get('Content-Type').match(/image\/(.*)/)[1]
        const imageUrl = `/images/${project.id}.${ext}`
        const thumb = await imageThumbnail(r.body, { height: 200, width: 290 })
        await fs.promises.writeFile('public' + imageUrl, thumb)
        project.image = imageUrl
      } catch (e) {
        console.error("Can't write thumbnail for:", image)
        console.error(e)
        delete project.image
      }
    }
  }
  return project
}

function shuffleArray(array) {
  for (let i = array.length - 1; i > 0; i--) {
    const j = Math.floor(Math.random() * (i + 1))
    ;[array[i], array[j]] = [array[j], array[i]] // eslint-disable-line no-param-reassign
  }
}