Merge branch 'gramhagen-pypi_prep'

.gitignore

@@ -57,6 +57,8 @@ test/RunTests.last.times
 *.dSYM
 *~
 *.pc
+*.pyc
+.DS_Store
 
 # build folders
 **/x64
@@ -98,16 +100,20 @@ VW.creator.user
 VW.files
 VW.includes
 
-/cs/Serializer/Visitors/VowpalWabbitNativeVisitorExt.cs
-# Python package
-build/**
-dist/**
-pyvw.egg-info/**
-python/pyvw.egg-info/**
+# Python packaging
+python/.cache/**
+python/.eggs/**
+python/.tox/**
+python/build/**
+python/dist/**
+python/src/**
+python/vowpalwabbit.egg-info/**
+python/.coverage
 
 /COPYING
 /README
 /compile
+/cs/Serializer/Visitors/VowpalWabbitNativeVisitorExt.cs
 /cs/Serializer/VowpalWabbitDefaultMarshallerExt.cs
 /cs_unittest/RunTests.cs
 vowpalwabbit/vw.VC.VC.opendb

.travis.yml

@@ -6,13 +6,27 @@ before_install:
   - sudo apt-get install -qq libboost-all-dev
   - sudo apt-get install maven
   - sudo pip install cpp-coveralls wheel
+  # use miniconda for python package testing
+  - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
+  - bash miniconda.sh -b -p $HOME/miniconda
+  - export PATH="$HOME/miniconda/bin:$PATH"
+  - hash -r
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  - conda create -q -n test-python27 python=2.7 nomkl numpy scipy scikit-learn
 script:
   - make all
   - make python
   - mvn test -f java/pom.xml
   - make test
   - make test_gcov --always-make
-  - python setup.py sdist
-  - python setup.py bdist_wheel
+  - cd python
+  - source activate test-python27
+  - pip install pytest readme_renderer
+  - python setup.py check -mrs
+  - python setup.py install
+  - py.test tests
+  - source deactivate
+  - cd ..
 after_success:
   - coveralls --exclude lib --exclude tests --gcov-options '\-lp'

python/MANIFEST.in

@@ -0,0 +1,4 @@
+include README.rst *.txt
+recursive-include src *
+recursive-include vowpalwabbit *
+global-exclude *.o *.exe *.pyc

python/Makefile

@@ -8,11 +8,19 @@
 #  $(error Please run 'make' at the top level only)
 #endif
 
-PYTHON_VERSION = 2.7
+# The python version and python boost library are resolved and set using setup.py
+# When running make directly PYTHON_VERSION and PYTHON_LIBS env vars can be set to change these defaults
+ifeq ($(PYTHON_VERSION),)
+  PYTHON_VERSION = 2.7
+endif
+
+ifeq ($(PYTHON_LIBS),)
+  PYTHON_LIBS = -l boost_python
+endif
+
 PYTHON_INCLUDE = $(shell python$(PYTHON_VERSION)-config --includes)
 PYTHON_LDFLAGS = $(shell python$(PYTHON_VERSION)-config --ldflags)
 
-PYTHON_LIBS = -l boost_python
 ifeq (,$(wildcard ../vowpalwabbit/vw.so))
   VWLIBS = ../vowpalwabbit/libvw.a ../vowpalwabbit/liballreduce.a
 else
@@ -26,7 +34,9 @@ all:
 things: $(PYLIBVW)
 
 ifeq "CYGWIN" "$(findstring CYGWIN,$(UNAME))"
-  PYTHON_LIBS = -l boost_python-mt
+  ifeq ($(PYTHON_LIBS),)
+    PYTHON_LIBS = -l boost_python-mt
+  endif
   PYLIBVW = pylibvw.dll
 endif
 ifneq ($(UNAME), Darwin)
@@ -39,6 +49,7 @@ $(PYLIBVW): pylibvw.o  ../vowpalwabbit/libvw.a
 	$(CXX) -shared $(EXPORT_DYNAMIC) pylibvw.o $(BOOST_LIBRARY) $(PYTHON_LDFLAGS) $(VWLIBS) $(STDLIBS) -o $(PYLIBVW)
 
 pylibvw.o: pylibvw.cc
+	$(info Using Python $(PYTHON_VERSION))
 	$(CXX) -std=c++0x $(PYTHON_INCLUDE) $(BOOST_INCLUDE) -fPIC -c pylibvw.cc -o pylibvw.o
 
 clean:

python/PACKAGE.rst

@@ -0,0 +1,70 @@
+Python Packaging
+================
+
+Setting up the python package for distribution on PyPI is done using commands through setup.py
+The following instructions assume you are working from the python directory.
+
+Deployment Process
+------------------
+
+0) Commit changes (increment the version in configure.ac file, PyPI will not overwrite a package using the same version)
+1) Update MANIFEST.in to include any additional files then check it to make sure the dist has the right data in it
+
+    .. code-block:: bash
+
+        $ check-manifest --ignore Makefile,PACKAGE.rst,*.cc,tox.ini,tests*,examples*,src*
+
+2) Lint the code:
+   .. code-block:: bash
+
+       $ pylint -f colorized vowpalwabbit
+
+3) Make sure code passes all tests under supported environments
+
+    .. code-block:: bash
+
+        $ python setup.py test
+
+
+4) Create dist folder for package
+
+    .. code-block:: bash
+
+        $ python setup.py sdist
+
+5) Upload package to PyPI
+
+    You should have twine installed and configured and your PyPI / test PyPI user should have access to the package
+    <VERSION> corresponds to the new version in the configure.ac file
+
+    a) Test package
+
+    .. code-block:: bash
+
+        $ twine upload -r test dist/*
+        $ cd /tmp
+        $ virtualenv test_vw_package
+        $ source test_vw_package/bin/activate
+        $ pip install -i https://testpypi.python.org/simple/ vowpalwabbit
+        $ python -c 'from vowpalwabbit import pyvw'
+        $ deactivate
+        $ rm -rf test_vw_package
+
+    b) Upload package
+
+    .. code-block:: bash
+
+        $ twine upload dist dist/vowpalwabbit-<VERSION>.tar.gz
+
+6) Cleanup build and packaging artifacts / directories
+
+    .. code-block:: bash
+
+        $ python setup.py clean
+
+References
+==========
+
+https://packaging.python.org/en/latest/distributing/
+
+http://setuptools.readthedocs.io/en/latest/setuptools.html

python/README.rst

@@ -0,0 +1,104 @@
+Vowpal Wabbit Python Wrapper
+============================
+
+Vowpal Wabbit is a fast machine learning library for online learning.
+
+Installation
+------------
+
+From PyPI:
+
+.. code-block:: bash
+
+    $ pip install vowpalwabbit
+
+From remote repo:
+
+.. code-block:: bash
+
+    $ pip install -e git+https://github.com/JohnLangford/vowpal_wabbit/python
+
+From local repo:
+
+.. code-block:: bash
+
+    $ cd python
+    $ python setup.py install
+
+or
+
+.. code-block::
+
+    $ cd python
+    $ pip install -e .
+
+Usage
+-----
+
+You can use the python wrapper directly like this:
+
+.. code-block:: python
+
+    >>> from vowpalwabbit impor pyvw
+    >>> vw = pyvw.vw(quiet=True)
+    >>> ex = vw.example('1 | a b c')
+    >>> vw.learn(ex)
+    >>> vw.predict(ex)
+
+Or you can use the scikit-learn interface like this:
+
+.. code-block:: python
+
+    >>> import numpy as np
+    >>> from sklearn import datasets
+    >>> from sklearn.cross_validation import train_test_split
+    >>> from vowpalwabbit.sklearn_vw import VWClassifier
+    >>>
+    >>> # generate some data
+    >>> X, y = datasets.make_hastie_10_2(n_samples=10000, random_state=1)
+    >>> X = X.astype(np.float32)
+    >>>
+    >>> # split train and test set
+    >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=256)
+    >>>
+    >>> # build model
+    >>> model = VWClassifier()
+    >>> model.fit(X_train, y_train)
+    >>>
+    >>> # evaluate model
+    >>> model.score(X_train, y_train)
+    >>> model.score(X_test, y_test))
+
+Development
+-----------
+
+Contributions are welcome for improving the python wrapper to Vowpal Wabbit.
+
+1. Check for open issues_ or create one to discuss a feature idea or bug.
+2. Fork the repo_ on Github and make changes to the master branch (or a new branch off of master).
+3. Write a test in the python/tests folder showing the bug was fixed or feature works (recommend using pytest_).
+4. Make sure package installs and tests pass under all supported environments (this calls tox_ automatically).
+5. Send the pull request.
+
+Tests can be run using setup.py:
+
+.. code-block:: bash
+
+    $ python setup.py test
+
+
+Directory Structure:
+
+* python : this is where the c++ extension lives
+* python/vowpalwabbit : this is then main directory for python wrapper code and utilities
+* python/examples : example python code and jupyter notebooks to demonstrate functionality
+* python/tests : contains all tests for python code
+
+**Note:** neither examples nor tests directories are included in the distributed package, they are only for development purposes.
+
+.. _issues: https://github.com/JohnLangford/vowpal_wabbit/issues
+.. _repo: https://github.com/JohnLangford/vowpal_wabbit
+.. _pytest: http://pytest.org/latest/getting-started.html
+.. _tox: https://tox.readthedocs.io/en/latest/index.html
+
+