[Airflow] airflow설치 방법(with ubuntu, mysql)

mysql 설치

# 시스템 업그레이드
sudo apt update && sudo apt upgrade -y
# mysql 서버 설치
sudo apt-get install -y mysql-server
# mysql enble 설정
sudo systemctl enable mysql
# mysql 서버 
sudo systemctl status mysql

● mysql.service - MySQL Community Server
     Loaded: loaded (/lib/systemd/system/mysql.service; enabled; vendor preset:>
     Active: active (running) since Sun 2022-09-04 16:05:53 KST; 5s ago
   Main PID: 3169 (mysqld)
     Status: "Server is operational"
      Tasks: 39 (limit: 5828)
     Memory: 358.6M
        CPU: 950ms
     CGroup: /system.slice/mysql.service
             └─3169 /usr/sbin/mysqld

 9월 04 16:05:52 hsjoo-VirtualBox systemd[1]: Starting MySQL Community Server...
 9월 04 16:05:53 hsjoo-VirtualBox systemd[1]: Started MySQL Community Server.

mysql-server 접속

# mysql 초기 비밀번호 설정
sudo /usr/bin/mysql -u root -p

Enter password:
Welcome to the MySQL monitor.  Commands end with ; or \\g.
Your MySQL connection id is 8
Server version: 8.0.30-0ubuntu0.22.04.1 (Ubuntu)

Copyright (c) 2000, 2022, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\\h' for help. Type '\\c' to clear the current input statement.

mysql>

Airflow용 계정 만들기

mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY '비밀번호';
mysql> create user 'airflow'@'localhost' identified by '비밀번호';
mysql> create user 'airflow'@'%' identified by '비밀번호';
mysql> CREATE DATABASE airflow;
mysql> exit

Airflow 디펜던시 설치

sudo apt-get install -y vim ssh openssh-server software-properties-common apt-transport-https wget
sudo apt-get install -y python3
sudo apt-get install -y freetds-bin
sudo apt-get install -y krb5-user
sudo apt-get install -y ldap-utils
sudo apt-get install -y libsasl2-2
sudo apt-get install -y libsasl2-modules
sudo apt-get install -y libssl-dev
sudo apt-get install -y locales
sudo apt-get install -y lsb-release
sudo apt-get install -y sqlite3
sudo apt-get install -y sasl2-bin
sudo apt-get install -y unixodbc 
sudo apt-get install -y python3-pip
sudo apt-get install -y python3-testresources
sudo apt-get install -y mysql-client
sudo apt-get install -y libmysqlclient-dev
sudo apt-get install -y python3-mysqldb

Airflow 환경 설정 적용

# 환경설정
export AIRFLOW_HOME=~/airflow
export AIRFLOW_VERSION=2.3.4
export PYTHON_VERSION="$(python3 --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
export CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
export PATH=$PATH:/home/호스트/.local/bin
# 설정 확인 
echo $AIRFLOW_HOME
/home/호스트/airflow

Airflow 설치

cd ~
mkdir airflow
# airflow 설치
pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
# mysql 모듈 설치
pip install apache-airflow[mysql]

Airflow 초기 설정 적용

# 초기 폴더와 파일 만들기
airflow db init

Airflow Home폴더 안에 초기 파일 생성 됨

total 468
drwxrwxr-x  3 hsjoo hsjoo   4096  9월  4 17:46 .
drwxr-x--- 17 hsjoo hsjoo   4096  9월  4 17:43 ..
-rw-rw-r--  1 hsjoo hsjoo  49389  9월  4 17:45 airflow.cfg
-rw-r--r--  1 hsjoo hsjoo 401408  9월  4 17:46 airflow.db
drwxrwxr-x  3 hsjoo hsjoo   4096  9월  4 17:45 logs
-rw-rw-r--  1 hsjoo hsjoo   4707  9월  4 17:45 webserver_config.py

Airflow 설정 변경

# DB mysql로 변경 
sudo vim airflow.cfg

# 아래 내용 변경
[CORE]
...
executor = LocalExecutor
...

[database]
...
# sql_alchemy_conn = mysql://[id]:[pw]@localhost:3306/[DB]
sql_alchemy_conn = mysql://airflow:airflow@localhost:3306/airflow
...

Airflow 에서 접근 DB  권한 설정

# mysql airflow DB 만들어진 테이블 접근 권한 부여
sudo mysql -u root -p
mysql> grant all privileges on airflow.* to airflow@localhost;
mysql> grant all privileges on airflow.* to airflow@'%';
mysql> flush privileges;
# db 초기화
airflow db init

Airflow 시작하기

# 초기 folder(dags, plugins)만들기 
cd $AIRFLOW_HOME
sudo mkdir ~/airflow/dags ~/airflow/plugins
# user create password는 나중에 쳐야함
airflow users create \
    --username airflow \
    --firstname airflow \
    --lastname airflow \
    --role Admin \
    --email airflow@example.org
# webserver 실행
airflow webserver --port 8080 -D
# scheduler 실행
airflow scheduler -D