Update and add test database setup

.gitignore

@@ -1,2 +1,3 @@
 .vagrant
 config.yaml
+puppet/modules

README.md

@@ -8,12 +8,14 @@ hi. This brings up a disposable Dreamwidth dev instance, with all required servi
 
 - Install [VirtualBox](https://www.virtualbox.org/).
 - Install [Vagrant](https://www.vagrantup.com/).
+  - You will probably need to install the [vagrant-vbguest](https://github.com/dotless-de/vagrant-vbguest) plugin as well
 - Fork [dreamwidth/dw-free](https://github.com/dreamwidth/dw-free/) to your own account.
 
 ### Run the thing
 
 - Clone this repo.
 - Copy `config-example.yaml` to `config.yaml` and edit as needed.
+- Install the puppet modules (see puppet/README.md for details)
 - Make sure you're not running on battery and are on an unmetered internet connection.
 - `vagrant up`
     - It'll ask you which host network interface to bridge to. Probably choose your main wifi interface, which on a modern laptop is usually listed as en0.
@@ -98,11 +100,8 @@ good gracious, please do.
 
 when iterating more quickly, the command you need for refreshing is `vagrant provision --provision-with puppet`
 
-All the Puppet modules are just vendored because I'm lazy and I assume you are too. `dw_dev` is the only unique one.
-
 IDK if bridged mode networking is actually good to be using here (spoiler, probably not), but I never did figure out how to master virtualbox's multi-adapter whoopie-cushion tornado. ugh. Maybe later.
 
 ## License(s)
 
-- Like I said, I vendored a lot of Puppet modules, so I guess I'm accidentally a distro now. They all ride under the terms of their original licenses, which are vendored right along with em. Mostly Apache 2, small GPL contingent, one Perl Artistic.
 - The stuff I wrote is all in the dw_dev module, which I'm making available under the Apache 2 licence.

Vagrantfile

@@ -15,8 +15,7 @@ end
 # backwards compatibility). Please don't change it unless you know what
 # you're doing.
 Vagrant.configure("2") do |config|
-  config.vm.box = "puppetlabs/ubuntu-16.04-64-puppet"
-  config.vm.box_version = "1.0.0"
+  config.vm.box = "debian/buster64"
 
   # Create a forwarded port mapping which allows access to a specific port
   # within the machine from a port on the host machine. In the example below,
@@ -39,6 +38,9 @@ Vagrant.configure("2") do |config|
   config.vm.network "public_network"
   config.vm.hostname = hostname
 
+  # Install puppet into the base image
+  config.vm.provision "shell", inline: "apt-get install --yes puppet"
+
   # Share an additional folder to the guest VM. The first argument is
   # the path on the host to the actual folder. The second argument is
   # the path on the guest to mount the folder. And the optional third
@@ -63,7 +65,6 @@ Vagrant.configure("2") do |config|
   end
 
   config.vm.provision "puppet", run: "always" do |pup|
-    pup.binary_path = "/opt/puppetlabs/bin"
     pup.environment_path = ["vm", "/etc/puppetlabs/code/environments"]
     pup.environment = "production"
   end

puppet/Puppetfile

@@ -0,0 +1,12 @@
+mod 'puppetlabs/apache'
+mod 'puppetlabs/concat'
+mod 'puppetlabs/mailalias_core'
+mod 'puppetlabs/mysql'
+mod 'puppetlabs/stdlib'
+mod 'puppetlabs/translate'
+mod 'puppetlabs/vcsrepo'
+
+mod 'alternatives', :git => 'https://github.com/voxpupuli/puppet-alternatives'
+mod 'augeas', :git => 'https://github.com/camptocamp/puppet-augeas'
+mod 'cpanm', :git => 'https://github.com/jamesmcdonald/puppet-cpanm'
+mod 'postfix', :git => 'https://github.com/camptocamp/puppet-postfix'

puppet/README.md

@@ -0,0 +1,24 @@
+The modules/ directory is maintained by [r10k](https://github.com/puppetlabs/r10k).
+
+When you first check out this repository, you need to populate the modules via:
+
+- Install r10k (it's a ruby gem, install via `gem install r10k`)
+- Run r10k to fetch the modules
+- Copy the `dw_dev` module into `modules`
+
+```
+$ gem install r10k
+$ r10k puppetfile install
+$ cp -R dw_dev modules
+```
+
+The last step is needed because r10k wants to manage the entire `modules/`
+directory itself and will delete anything not in the `Puppetfile`; copying in
+the directory is simpler than maintaining another git repo for it.
+
+To update modules, repeat the last two steps (or run update.sh).
+
+NOTE: If you are developing `dw-dev-vagrant` itself, and are running linux, you
+could symlink rather than copying the `dw_dev` directory - it will prevent you
+from accidentally changing the copy in `modules/` and having r10k delete your
+changes.

puppet/modules/dw_dev/manifests/app.pp → puppet/dw_dev/manifests/app.pp

@@ -103,6 +103,17 @@
     group => $dw_user,
   }
 
+  # chkconfig.pl fails if this file is not present.
+  # Set to always copy over the file, since the comments say not to override it
+  # in place.
+  file {'config.pl':
+    path => "${ljhome}/etc/config.pl",
+    ensure => file,
+    source => "${ljhome}/etc/config.pl.example",
+    owner => $dw_user,
+    group => $dw_user,
+  }
+
   # The actual config files. I recommend editing the templates when you need to
   # change settings, so they'll survive across both restarts and the occasional
   # total dev environment wipe. But if you'd rather not, you can set the

puppet/modules/dw_dev/manifests/init.pp → puppet/dw_dev/manifests/init.pp

@@ -45,10 +45,18 @@
     keep_local_config_edits => $keep_local_config_edits,
   }
 
+  class {'dw_dev::test':
+    dw_user => $dw_user,
+    ljhome => $ljhome,
+    dw_db_user => $dw_db_user,
+    dw_db_user_password => $dw_db_user_password,
+  }
+
   contain([
     Class['dw_dev::prerequisites'],
     Class['dw_dev::user'],
     Class['dw_dev::app'],
+    Class['dw_dev::test'],
   ])
 
   ## The worker manager service:

puppet/modules/dw_dev/manifests/prerequisites.pp → puppet/dw_dev/manifests/prerequisites.pp

@@ -27,6 +27,7 @@
     'unzip',
     'links',
     'vim',
+    'libpng-dev',
     'libclass-autouse-perl',
     'libdatetime-perl',
     'libcache-memcached-perl',
@@ -42,6 +43,7 @@
     'libclass-data-inheritable-perl',
     'libclass-trigger-perl',
     'libcrypt-dh-perl',
+    'liblocale-codes-perl',
     'libmath-bigint-gmp-perl',
     'liburi-fetch-perl',
     'libgd-graph-perl',
@@ -190,9 +192,24 @@
     'Paws::S3',
     'Net::DNS',
     'Text::Fuzzy',
-    'JSON::Validator',
-    'Perl::Tidy',
+    'JSON::Validator@3.25',
+    'Perl::Tidy@20190601',
     'Test::Code::TidyAll',
+    'Authen::OATH',
+    'Authen::Passphrase::BlowfishCrypt',
+    'Authen::Passphrase::Clear',
+    'Convert::Base32',
+    'CryptX',
+    'Imager::File::PNG',
+    'Imager::QRCode',
+    'Log::Log4perl::Appender::Elasticsearch::Bulk',
+    'Math::Random::Secure',
+    'Net::Subnet',
+    'Sphinx::Search',
+    'Test::MockObject',
+    'UUID::Tiny',
+    '[email protected]',
+    'Captcha::[email protected]',
   ]
 
   include cpanm

puppet/dw_dev/manifests/test.pp

@@ -0,0 +1,113 @@
+class dw_dev::test (
+  String $dw_user = 'dw',
+  String $ljhome = '/home/dw/dw',
+  String $dw_db_user = 'dw',
+  String $dw_db_user_password,
+){
+
+  file {"${ljhome}/ext/local/t":
+    ensure => directory,
+    owner => $dw_user,
+    group => $dw_user,
+  }
+
+  file {'config-test-private.pl':
+    path => "${ljhome}/ext/local/t/config-test-private.pl",
+    ensure => file,
+    content => epp('dw_dev/config-test-private.epp', {
+	'dw_db_user' => $dw_db_user,
+	'dw_db_user_password' => $dw_db_user_password,
+      }),
+    owner => $dw_user,
+    group => $dw_user,
+    replace => true,
+  }
+
+  file {'config-test.pl':
+    path => "${ljhome}/ext/local/t/config-test.pl",
+    ensure => file,
+    content => epp('dw_dev/config-test.epp', {}),
+    owner => $dw_user,
+    group => $dw_user,
+    replace => true,
+  }
+
+  file {'populate-test-db':
+    path => "${ljhome}/ext/local/t/populate-test-db",
+    ensure => file,
+    content => epp('dw_dev/populate-test-db.epp', {}),
+    owner => $dw_user,
+    group => $dw_user,
+    replace => true,
+    mode => '0755',
+    require => Vcsrepo['dw-free'],
+  }
+
+  ## Test databases:
+  mysql::db {'test_master':
+    user => $dw_db_user,
+    password => $dw_db_user_password,
+    host => 'localhost',
+    grant => 'ALL',
+  }
+  mysql::db {'test_schwartz':
+    user => $dw_db_user,
+    password => $dw_db_user_password,
+    host => 'localhost',
+    grant => 'ALL',
+  }
+
+  exec {
+    default:
+      environment => ["LJHOME=${ljhome}", "DW_TEST=1"],
+      cwd => $ljhome,
+      user => $dw_user,
+      logoutput => true,
+    ;
+
+    'populate-test-db':
+      command => "${ljhome}/ext/local/t/populate-test-db",
+      refreshonly => true,
+      subscribe => [
+        Mysql::Db['test_master'],
+        Mysql::Db['test_schwartz'],
+        File['populate-test-db'],
+      ],
+    ;
+
+    # This follows the flow of t/bin/initialize-db and should ideally replace
+    # the populate-test-db shellscript above.
+    #
+    # 'update-test-db once':
+    #   command => "${ljhome}/bin/upgrading/update-db.pl -r --innodb",
+    #   refreshonly => true,
+    #   subscribe => [
+    #     Mysql::Db['test_master'],
+    #     Mysql::Db['test_schwartz'],
+    #     File['config-test.pl'],
+    #     File['config-test-private.pl'],
+    #   ],
+    # ;
+    # 'update-test-db twice':
+    #   command => "${ljhome}/bin/upgrading/update-db.pl -r --innodb",
+    #   refreshonly => true,
+    #   subscribe => Exec['update-test-db once'],
+    # ;
+    # 'update-cluster once':
+    #   command => "${ljhome}/bin/upgrading/update-db.pl -r --cluster=all --innodb",
+    #   refreshonly => true,
+    #   subscribe => Exec['update-test-db twice'],
+    # ;
+    # 'update-cluster twice':
+    #   command => "${ljhome}/bin/upgrading/update-db.pl -r --cluster=all --innodb",
+    #   refreshonly => true,
+    #   subscribe => Exec['update-cluster once'],
+    # ;
+    # 'texttool':
+    #   command => "${ljhome}/bin/upgrading/texttool.pl load",
+    #   refreshonly => true,
+    #   subscribe => Vcsrepo['dw-free'], # only needs to run on code updates.
+    #   # if you're handling code updates yourself, you're in charge of this.
+    # ;
+  }
+}

puppet/dw_dev/templates/config-test-private.epp

@@ -0,0 +1,23 @@
+<%- |
+  String $dw_db_user_password,
+  String $dw_db_user,
+| -%>
+{
+    ### Copy this file into ext/local/t to contain custom database information
+    package DW::PRIVATE;
+
+    %DBINFO = (
+        master => {
+            dbname => "test_master",
+            user => "<%= $dw_db_user %>",
+            pass => "<%= $dw_db_user_password %>",
+        },
+        theschwartz => {
+            dbname => "test_schwartz",
+            user => "<%= $dw_db_user %>",
+            pass => "<%= $dw_db_user_password %>",
+        }
+    );
+}
+
+1;