airflow快速安装

快速安装

# airflow 需要 home 目录，默认是~/airflow，
# 但是如果你需要，放在其它位置也是可以的
# (可选)
> export AIRFLOW_HOME = ~/airflow

# 使用 pip 从 pypi 安装
> pip3 install apache-airflow
Installing collected packages: gunicorn, werkzeug, typing-extensions, iso8601, tabulate, idna, dnspython, email-validator, lazy-object-proxy, unicodecsv, zope.deprecation, six, tenacity, numpy, python-dateutil, pytz, pandas, sqlalchemy, typing, sqlalchemy-jsonfield, pygments, zipp, importlib-metadata, argcomplete, lockfile, docutils, python-daemon, MarkupSafe, Mako, python-editor, alembic, jinja2, itsdangerous, click, flask, wtforms, flask-admin, colorlog, text-unidecode, python-slugify, python-nvd3, graphviz, configparser, tzlocal, PyYAML, flask-swagger, future, attrs, cattrs, cached-property, dill, chardet, certifi, urllib3, requests, json-merge-patch, natsort, croniter, psutil, flask-caching, flask-login, flask-wtf, pyrsistent, jsonschema, thrift, marshmallow, marshmallow-enum, PyJWT, marshmallow-sqlalchemy, apispec, sqlalchemy-utils, prison, defusedxml, python3-openid, Flask-OpenID, Flask-SQLAlchemy, Flask-JWT-Extended, Babel, Flask-Babel, colorama, flask-appbuilder, markdown, funcsigs, setproctitle, pytzdata, pendulum, apache-airflow
  Running setup.py install for unicodecsv ... done
  Running setup.py install for typing ... done
  Running setup.py install for alembic ... done
  Running setup.py install for flask-admin ... done
  Running setup.py install for python-slugify ... done
  Running setup.py install for python-nvd3 ... done
  Running setup.py install for tzlocal ... done
  Running setup.py install for PyYAML ... done
  Running setup.py install for flask-swagger ... done
  Running setup.py install for future ... done
  Running setup.py install for dill ... done
  Running setup.py install for json-merge-patch ... done
  Running setup.py install for psutil ... error
    Complete output from command /usr/bin/python3 -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-rbpbx5v6/psutil/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-y1gdilml-record/install-record.txt --single-version-externally-managed --compile:
    running install
    running build
    running build_py
    creating build
    creating build/lib.linux-x86_64-3.6
    creating build/lib.linux-x86_64-3.6/psutil
    copying psutil/_pswindows.py -> build/lib.linux-x86_64-3.6/psutil
    copying psutil/_psbsd.py -> build/lib.linux-x86_64-3.6/psutil
    copying psutil/_compat.py -> build/lib.linux-x86_64-3.6/psutil
    copying psutil/_common.py -> build/lib.linux-x86_64-3.6/psutil
    copying psutil/_psposix.py -> build/lib.linux-x86_64-3.6/psutil
    copying psutil/__init__.py -> build/lib.linux-x86_64-3.6/psutil
    copying psutil/_psaix.py -> build/lib.linux-x86_64-3.6/psutil
    copying psutil/_pslinux.py -> build/lib.linux-x86_64-3.6/psutil
    copying psutil/_pssunos.py -> build/lib.linux-x86_64-3.6/psutil
    copying psutil/_psosx.py -> build/lib.linux-x86_64-3.6/psutil
    creating build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_aix.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_system.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_testutils.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_bsd.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_connections.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_memleaks.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_unicode.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_posix.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_windows.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_misc.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_sunos.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/__init__.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_osx.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_process.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_linux.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/__main__.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/test_contracts.py -> build/lib.linux-x86_64-3.6/psutil/tests
    copying psutil/tests/runner.py -> build/lib.linux-x86_64-3.6/psutil/tests
    running build_ext
    building 'psutil._psutil_linux' extension
    creating build/temp.linux-x86_64-3.6
    creating build/temp.linux-x86_64-3.6/psutil
    gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -fPIC -DPSUTIL_POSIX=1 -DPSUTIL_SIZEOF_PID_T=4 -DPSUTIL_VERSION=572 -DPSUTIL_LINUX=1 -I/usr/include/python3.6m -c psutil/_psutil_common.c -o build/temp.linux-x86_64-3.6/psutil/_psutil_common.o
    psutil/_psutil_common.c:9:20: fatal error: Python.h: No such file or directory
     #include <Python.h>
                        ^
    compilation terminated.
    error: command 'gcc' failed with exit status 1
    
    ----------------------------------------
Command "/usr/bin/python3 -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-rbpbx5v6/psutil/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-y1gdilml-record/install-record.txt --single-version-externally-managed --compile" failed with error code 1 in /tmp/pip-build-rbpbx5v6/psutil/

升级pip版本

> pip3 install—upgrade pip
> pip3 install apache-airflow
Installing collected packages: thrift, setproctitle, psutil, apache-airflow
    Running setup.py install for thrift ... done
    Running setup.py install for setproctitle ... error
    ERROR: Command errored out with exit status 1:
     command: /usr/bin/python3 -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'/tmp/pip-install-o6mdhdgg/setproctitle/setup.py'"'"'; __file__='"'"'/tmp/pip-install-o6mdhdgg/setproctitle/setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record /tmp/pip-record-96flcmqf/install-record.txt --single-version-externally-managed --compile --install-headers /usr/local/include/python3.6m/setproctitle
         cwd: /tmp/pip-install-o6mdhdgg/setproctitle/
    Complete output (15 lines):
    running install
    running build
    running build_ext
    building 'setproctitle' extension
    creating build
    creating build/temp.linux-x86_64-3.6
    creating build/temp.linux-x86_64-3.6/src
    gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -fPIC -DSPT_VERSION=1.1.10 -DHAVE_SYS_PRCTL_H=1 -I/usr/include/python3.6m -c src/setproctitle.c -o build/temp.linux-x86_64-3.6/src/setproctitle.o
    In file included from src/spt.h:15:0,
                     from src/setproctitle.c:14:
    src/spt_python.h:14:20: fatal error: Python.h: No such file or directory
     #include <Python.h>
                        ^
    compilation terminated.
    error: command 'gcc' failed with exit status 1
    ----------------------------------------
ERROR: Command errored out with exit status 1: /usr/bin/python3 -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'/tmp/pip-install-o6mdhdgg/setproctitle/setup.py'"'"'; __file__='"'"'/tmp/pip-install-o6mdhdgg/setproctitle/setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record /tmp/pip-record-96flcmqf/install-record.txt --single-version-externally-managed --compile --install-headers /usr/local/include/python3.6m/setproctitle Check the logs for full command output.

“Python.h: No such file or directory”报错很明确，缺少python的引用文件

> yum install python3-devel
> yum install libevent-devel
> pip3 install apache-airflow
Successfully installed apache-airflow-1.10.12 psutil-5.7.2 setproctitle-1.1.10

注意python3-devel的版本，安装python-devel并不能引用

初始化数据库，默认采用本地化sqlite数据库

> airflow initdb

启动web服务器

> airflow webserver -p 8080

webAdmin.png

启动调度器

> airflow scheduler

在浏览器中浏览 localhost:8080，并在 home 页开启 example dag