Refactor, init command, auto-create version tables

This commit is contained in:
Juan Ferrer 2020-11-19 13:51:27 +01:00
parent fa4754fc09
commit 61de39aab9
33 changed files with 208 additions and 123 deletions

2
.gitignore vendored
View File

@ -1,4 +1,2 @@
.DS_Store
node_modules
db.*.ini
.procs-priv.sql

View File

@ -16,7 +16,11 @@ RUN apt-get update \
COPY \
myvc-dump.sh \
myvc-push.sh \
db.ini \
structure.sql \
/usr/local/bin/
COPY \
workspace/remotes/local.ini \
/usr/local/bin/db.ini
WORKDIR /workspace

View File

@ -3,25 +3,25 @@ FROM myvc/server
USER root
COPY \
dump/.dump.sql \
dump/structure.sql \
myvc.config.json \
myvc.structure.sql \
.dump.sql \
./
RUN gosu mysql docker-init.sh \
&& docker-dump.sh myvc.structure \
&& docker-dump.sh .dump \
&& docker-dump.sh dump/structure \
&& docker-dump.sh dump/.dump \
&& gosu mysql docker-temp-stop.sh
COPY routines ./routines
COPY changes ./changes
COPY myvc.fixtures.sql ./
COPY versions ./versions
COPY dump/fixtures.sql ./
ARG STAMP=unknown
RUN gosu mysql docker-temp-start.sh \
&& myvc-push.sh \
&& docker-dump.sh myvc.fixtures \
&& myvc-push.sh -a \
&& docker-dump.sh dump/fixtures \
&& gosu mysql docker-temp-stop.sh
RUN echo "[INFO] -> Import finished" \
&& rm -rf /docker-boot
RUN echo "[LOG] Import finished." \
&& rm -rf /workspace
USER mysql

View File

@ -9,7 +9,10 @@ RUN apt-get update \
&& curl -sL https://apt.verdnatura.es/conf/verdnatura.gpg | apt-key add - \
&& echo "deb http://apt.verdnatura.es/ jessie main" > /etc/apt/sources.list.d/vn.list \
&& apt-get update \
&& apt-get install -y vn-mariadb \
&& apt-get install -y \
git \
jq \
vn-mariadb \
&& apt-get purge -y --auto-remove curl ca-certificates \
&& rm -rf /var/lib/apt/lists/*
@ -21,13 +24,14 @@ COPY \
docker/docker-dump.sh \
docker/docker-start.sh \
myvc-push.sh \
structure.sql \
db.ini \
/usr/local/bin/
RUN mkdir /mysql-data \
&& chown -R mysql:mysql /mysql-data
WORKDIR /docker-boot
COPY db.ini ./
WORKDIR /workspace
USER mysql
ENTRYPOINT ["docker-start.sh"]

View File

@ -45,7 +45,8 @@ otherwise indicated, the default environment is *production*.
Commands for database versioning:
* **pull**: Exports database routines into workspace.
* **init**: Initialize an empty workspace.
* **pull**: Export database routines into workspace.
* **push**: Apply changes into database, uses *test* environment by default.
Commands for local server management:
@ -58,31 +59,33 @@ Each command can have its own specific commandline options.
## Basic information
First of all you have to import *structure.sql* into your database. This script
includes the tables where MyVC stores information about applied versions.
First of all you have to initalize your workspace.
Create *myvc.config.json* main configuration file at the root of your project
folder, this file should include the project codename and schemas/tables wich
are exported when you use *pull* or *dump* commands. You have an example of a
configuration file in the root folder of this project.
```text
$ myvc init
```
Now yoy can configure MyVC using *myvc.config.json* file, located at the root of
your workspace. This file should include the project codename and schemas/tables
wich are exported when you use *pull* or *dump* commands.
### Environments
Create database connection configuration for each environment at main project
Create database connection configuration for each environment at *remotes*
folder using standard MySQL *ini* configuration files. The predefined
environment names are *production* and *test*.
```text
db.[environment].ini
remotes/[environment].ini
```
### Dumps
Structure and fixture dumps will be created into hidden file *.dump.sql*. You
can also create your local fixture and structure files.
Structure and fixture dumps will be created into hidden file *dump/.dump.sql*.
You can also create your local fixture and structure files.
* *myvc.structure.sql*
* *myvc.fixtures.sql*
* *dump/structure.sql*
* *dump/fixtures.sql*
### Routines
@ -107,11 +110,11 @@ triggers and views with the following structure.
### Versions
Versions should be placed inside *changes* folder with the following structure.
!Don't place your PL/SQL objects here, use the routines folder!
Versions should be placed inside *versions* folder with the following structure.
Don't place your PL/SQL objects here, use the routines folder!
```text
changes
versions
|- 00001-firstVersionCodeName
| |- 00-firstExecutedScript.sql
| |- 01-secondScript.sql

2
db.ini
View File

@ -1,5 +1,5 @@
[client]
host = 172.17.0.1
host = localhost
port = 3306
user = root
password = root

View File

@ -1,6 +1,9 @@
#!/bin/bash
export MYSQL_PWD=root
FILE="/docker-boot/$1.sql"
echo "[INFO] -> Importing $FILE"
mysql -u root --default-character-set=utf8 --comments -f < "$FILE"
FILE="$1.sql"
if [ -f "$FILE" ]; then
echo "[LOG] -> Importing $FILE"
export MYSQL_PWD=root
mysql -u root --default-character-set=utf8 --comments -f < "$FILE"
fi

View File

@ -41,8 +41,6 @@ const commandArgs = {
applyUncommited: 'a'
},
default: {
force: false,
user: false,
env: 'test'
}
}
@ -65,7 +63,7 @@ parameter('Workspace:', opts.workspace);
parameter('Command:', command);
class MyVC {
async init(opts) {
async load(opts) {
// Configuration file
const configFile = 'myvc.config.json';
@ -82,7 +80,7 @@ class MyVC {
let iniFile = 'db.ini';
let iniDir = __dirname;
if (opts.env) {
iniFile = `db.${opts.env}.ini`;
iniFile = `remotes/${opts.env}.ini`;
iniDir = opts.workspace;
}
const iniPath = path.join(iniDir, iniFile);
@ -116,6 +114,16 @@ class MyVC {
});
}
async init(opts) {
const templateDir = `${__dirname}/workspace`;
const templates = await fs.readdir(templateDir);
for (let template of templates){
const dst = `${opts.workspace}/${template}`;
if (!await fs.pathExists(dst))
await fs.copy(`${templateDir}/${template}`, dst);
}
}
async pull(opts) {
const pull = require('./myvc-pull');
await pull(
@ -163,8 +171,8 @@ class MyVC {
try {
const myvc = new MyVC();
if (myvc[command]) {
await myvc.init(opts);
if (command != 'load' && myvc[command]) {
await myvc.load(opts);
await myvc[command](opts);
} else
throw new Error (`Unknown command '${command}'`);

View File

@ -3,11 +3,14 @@ set -e
CONFIG_FILE=$1
INI_FILE=$2
DUMP_FILE=".dump.sql"
DUMP_DIR="dump"
DUMP_FILE="$DUMP_DIR/.dump.sql"
echo "SELECT 1;" | mysql --defaults-file="$INI_FILE" >> /dev/null
SCHEMAS=( $(jq -r ".schemas[]" "$CONFIG_FILE") )
mkdir -p "$DUMP_DIR"
mysqldump \
--defaults-file="$INI_FILE" \
--default-character-set=utf8 \

View File

@ -9,7 +9,7 @@ class Exporter {
this.callback = callback;
this.dstDir = `${objectName}s`;
const templateDir = `${__dirname}/templates/${objectName}`;
const templateDir = `${__dirname}/exporters/${objectName}`;
this.query = fs.readFileSync(`${templateDir}.sql`, 'utf8');
const templateFile = fs.readFileSync(`${templateDir}.ejs`, 'utf8');

View File

@ -5,9 +5,18 @@ IS_USER=FALSE
APPLY_UNCOMMITED=FALSE
WORKSPACE="$PWD"
usage() {
echo "[ERROR] Usage: $0 [-f] [-u] [-a] [-e environment]"
exit 1
error() {
local MESSAGE=$1
>&2 echo "[ERR] $MESSAGE"
exit 1
}
warn() {
local MESSAGE=$1
>&2 echo "[WAR] $MESSAGE"
}
log() {
local MESSAGE=$1
echo "[LOG] $MESSAGE"
}
while getopts ":fuae:" option
@ -26,7 +35,7 @@ do
APPLY_UNCOMMITED=TRUE
;;
\?|:)
usage
error "Usage: $0 [-f] [-u] [-a] [-e environment]"
;;
esac
done
@ -38,8 +47,7 @@ shift $(($OPTIND - 1))
CONFIG_FILE="myvc.config.json"
if [ ! -f "$CONFIG_FILE" ]; then
echo "[ERROR] Config file not found: $CONFIG_FILE"
exit 2
error "Config file not found: $CONFIG_FILE"
fi
DIR="$(dirname "${BASH_SOURCE[0]}")"
@ -48,26 +56,36 @@ CODE=$(jq -r ".code" "$CONFIG_FILE")
# Load database configuration
if [ -z "$ENV" ]; then
INI_FILE="$WORKSPACE/db.ini"
INI_FILE="$DIR/db.ini"
else
INI_FILE="$WORKSPACE/db.$ENV.ini"
INI_FILE="$WORKSPACE/remotes/$ENV.ini"
fi
if [ ! -f "$INI_FILE" ]; then
echo "[ERROR] Database config file not found: $INI_FILE"
exit 2
error "Database config file not found: $INI_FILE"
fi
echo "[INFO] Using config file: $INI_FILE"
log "Using config file: $INI_FILE"
echo "SELECT 1;" | mysql --defaults-file="$INI_FILE" >> /dev/null
if [ "$?" -ne "0" ]; then
exit 3
error "Cannot connect to database."
fi
# Fetch git information
if [ ! -d "$WORKSPACE/.git" ]; then
error "Git directory not initialized."
fi
COMMIT_SHA=$(git rev-parse HEAD)
if [ "$?" -ne "0" ]; then
error "Cannot fetch Git HEAD."
fi
log "HEAD: $COMMIT_SHA"
git diff-index --quiet --cached HEAD --
STAGED=$?
@ -78,65 +96,86 @@ UNTRACKED=`git ls-files --others --exclude-standard`
if [ "$STAGED" == "1" ] || [ "$CHANGED" == "1" ] || [ -n "$UNTRACKED" ]; then
if [ "$APPLY_UNCOMMITED" == "TRUE" ]; then
echo "[WARN] You are applying uncommited changes."
warn "You are applying uncommited changes."
else
echo "[ERROR] You have uncommited changes, commit them before pushing or use -a option."
exit 2
error "You have uncommited changes, commit them before pushing or use -a option."
fi
fi
COMMIT_SHA=$(git rev-parse HEAD)
echo "[INFO] HEAD: $COMMIT_SHA"
# Query functions
dbQuery() {
SQL=$1
RETVAL=`echo "$SQL" | mysql --defaults-file="$INI_FILE" --silent --raw`
local SQL=$1
local SCHEMA=$2
RETVAL=`echo "$SQL" | mysql --defaults-file="$INI_FILE" --silent --raw "$SCHEMA"`
}
dbExec() {
SQL=$1
echo "$SQL" | mysql --defaults-file="$INI_FILE"
local SQL=$1
local SCHEMA=$2
echo "$SQL" | mysql --defaults-file="$INI_FILE" "$SCHEMA"
}
dbExecFromFile() {
FILE_PATH=$1
SCHEMA=$2
local FILE_PATH=$1
local SCHEMA=$2
mysql --defaults-file="$INI_FILE" --default-character-set=utf8 --comments "$SCHEMA" < $FILE_PATH
}
# Fetch database version
dbQuery "SELECT number, gitCommit FROM util.version WHERE code = '$CODE'"
VERSION_SCHEMA=$(jq -r ".versionSchema" "$CONFIG_FILE")
if [ "$VERSION_SCHEMA" == "null" ]; then
VERSION_SCHEMA="myvc"
fi
read -r -d '' SQL << EOM
SELECT COUNT(*)
FROM information_schema.tables
WHERE TABLE_SCHEMA = '$VERSION_SCHEMA'
AND TABLE_NAME = 'version'
EOM
dbQuery "$SQL"
TABLE_EXISTS=$RETVAL
SCHEMA="\`$VERSION_SCHEMA\`"
if [ "$TABLE_EXISTS" -eq "0" ]; then
dbExec "CREATE DATABASE IF NOT EXISTS $SCHEMA"
dbExecFromFile "$DIR/structure.sql" "$VERSION_SCHEMA"
log "Version tables created into $SCHEMA schema."
fi
dbQuery "SELECT number, gitCommit FROM $SCHEMA.version WHERE code = '$CODE'"
RETVAL=($RETVAL)
DB_VERSION=${RETVAL[0]}
DB_COMMIT=${RETVAL[1]}
echo "[INFO] Database information:"
echo "[INFO] -> Version: $DB_VERSION"
echo "[INFO] -> Commit: $DB_COMMIT"
log "Database information:"
log " -> Version: $DB_VERSION"
log " -> Commit: $DB_COMMIT"
if [[ ! "$DB_VERSION" =~ ^[0-9]*$ ]]; then
echo "[ERROR] Wrong database version."
exit 4
error "Wrong database version."
fi
if [[ -z "$DB_VERSION" ]]; then
if [ -z "$DB_VERSION" ]; then
DB_VERSION=00000
fi
if [ "$IS_USER" == "TRUE" ]; then
echo "[INFO] User information:"
log "User information:"
dbQuery "SELECT LEFT(USER(), INSTR(USER(), '@') - 1)"
DB_USER=$RETVAL
echo "[INFO] -> Name: $DB_USER"
log " -> Name: $DB_USER"
dbQuery "SELECT number, gitCommit FROM util.versionUser WHERE code = '$CODE' AND user = '$DB_USER'"
dbQuery "SELECT number, gitCommit FROM $SCHEMA.versionUser WHERE code = '$CODE' AND user = '$DB_USER'"
RETVAL=($RETVAL)
USER_VERSION=${RETVAL[0]}
USER_COMMIT=${RETVAL[1]}
echo "[INFO] -> Version: $USER_VERSION"
echo "[INFO] -> Commit: $USER_COMMIT"
log " -> Version: $USER_VERSION"
log " -> Commit: $USER_COMMIT"
if [ ! -z "$USER_VERSION" ]; then
if [ "$USER_VERSION" -gt "$DB_VERSION" ]; then
@ -161,24 +200,24 @@ if [ "$ENV" == "production" ]; then
echo ""
if [ "$FORCE" != "TRUE" ]; then
read -p "[INTERACTIVE] Are you sure? (Default: no) [yes|no]: " ANSWER
read -p "[INT] Are you sure? (Default: no) [yes|no]: " ANSWER
if [ "$ANSWER" != "yes" ]; then
echo "[INFO] Aborting changes."
log "Aborting changes."
exit
fi
fi
fi
# Apply changes
# Apply versions
N_CHANGES=0
CHANGES_DIR="$WORKSPACE/changes"
VERSIONS_DIR="$WORKSPACE/versions"
if [ -d "$CHANGES_DIR" ]; then
if [ -d "$VERSIONS_DIR" ]; then
LAST_APPLIED_VERSION=$DB_VERSION
for DIR_PATH in "$CHANGES_DIR/"*; do
for DIR_PATH in "$VERSIONS_DIR/"*; do
DIR_NAME=$(basename $DIR_PATH)
DIR_VERSION=${DIR_NAME:0:5}
@ -186,15 +225,15 @@ if [ -d "$CHANGES_DIR" ]; then
continue
fi
if [[ ! "$DIR_NAME" =~ ^[0-9]{5}(-[a-zA-Z0-9]+)?$ ]]; then
echo "[WARN] Ignoring wrong directory name: $DIR_NAME"
warn "Ignoring wrong directory name: $DIR_NAME"
continue
fi
if [ "$DB_VERSION" -ge "$DIR_VERSION" ]; then
echo "[INFO] Ignoring already applied version: $DIR_NAME"
log "Ignoring already applied version: $DIR_NAME"
continue
fi
echo "[INFO] Applying version: $DIR_NAME"
log "Applying version: $DIR_NAME"
for FILE in "$DIR_PATH/"*; do
FILE_NAME=$(basename "$FILE")
@ -203,11 +242,11 @@ if [ -d "$CHANGES_DIR" ]; then
continue
fi
if [[ ! "$FILE_NAME" =~ ^[0-9]{2}-[a-zA-Z0-9_]+\.sql$ ]]; then
echo "[WARN] Ignoring wrong file name: $FILE_NAME"
warn "Ignoring wrong file name: $FILE_NAME"
continue
fi
echo "[INFO] -> $FILE_NAME"
log " -> $FILE_NAME"
dbExecFromFile "$FILE"
N_CHANGES=$((N_CHANGES + 1))
done
@ -228,7 +267,7 @@ applyRoutines() {
continue
fi
if [[ ! "$FILE_NAME" =~ ^[a-zA-Z0-9_]+\.sql$ ]]; then
echo "[WARN] Ignoring wrong file name: $FILE_NAME"
warn "Ignoring wrong file name: $FILE_NAME"
continue
fi
@ -257,7 +296,7 @@ applyRoutines() {
ROUTINE_TYPE=VIEW
;;
*)
echo "[WARN] Ignoring unknown routine type: $ROUTINE_TYPE"
warn "Ignoring unknown routine type: $ROUTINE_TYPE"
continue
;;
esac
@ -270,7 +309,7 @@ applyRoutines() {
ACTION="DROP"
fi
echo "[INFO] -> $ACTION: $ROUTINE_TYPE $ROUTINE_NAME"
log " -> $ACTION: $ROUTINE_TYPE $ROUTINE_NAME"
if [ "$ACTION" == "REPLACE" ]; then
dbExecFromFile "$FILE_PATH" "$SCHEMA"
@ -286,7 +325,7 @@ ROUTINES_CHANGED=0
ROUTINES_DIR="$WORKSPACE/routines"
if [ -d "$ROUTINES_DIR" ]; then
echo "[INFO] Applying changed routines."
log "Applying changed routines."
PROCS_FILE=.procs-priv.sql
mysqldump \
@ -296,7 +335,7 @@ if [ -d "$ROUTINES_DIR" ]; then
--insert-ignore \
mysql procs_priv > "$PROCS_FILE"
if [[ -z "$DB_COMMIT" ]]; then
if [ -z "$DB_COMMIT" ]; then
applyRoutines "find routines -type f"
else
applyRoutines "git diff --name-only --diff-filter=D $DB_COMMIT -- routines"
@ -310,12 +349,12 @@ if [ -d "$ROUTINES_DIR" ]; then
dbExec "FLUSH PRIVILEGES"
rm "$PROCS_FILE"
else
echo "[WARN] An error ocurred when restoring routine privileges, backup saved at $PROCS_FILE"
warn "An error ocurred when restoring routine privileges, backup saved at $PROCS_FILE"
fi
echo "[INFO] -> $ROUTINES_CHANGED routines have changed."
log " -> $ROUTINES_CHANGED routines have changed."
else
echo "[INFO] -> No routines changed."
log " -> No routines changed."
rm "$PROCS_FILE"
fi
fi
@ -327,7 +366,7 @@ N_CHANGES=$((N_CHANGES + ROUTINES_CHANGED))
if [ "$N_CHANGES" -gt "0" ]; then
if [ "$IS_USER" == "TRUE" ]; then
SQL=(
"INSERT INTO util.versionUser SET "
"INSERT INTO $SCHEMA.versionUser SET "
"code = '$CODE', "
"user = '$DB_USER', "
"number = '$LAST_APPLIED_VERSION', "
@ -338,7 +377,7 @@ if [ "$N_CHANGES" -gt "0" ]; then
)
else
SQL=(
"INSERT INTO util.version SET "
"INSERT INTO $SCHEMA.version SET "
"code = '$CODE', "
"number = '$LAST_APPLIED_VERSION', "
"gitCommit = '$COMMIT_SHA' "
@ -349,7 +388,7 @@ if [ "$N_CHANGES" -gt "0" ]; then
fi
dbExec "${SQL[*]}"
echo "[INFO] Changes applied succesfully."
log "Changes applied succesfully."
else
echo "[INFO] No changes applied."
log "No changes applied."
fi

View File

@ -1,6 +1,6 @@
{
"name": "myvc",
"version": "1.0.17",
"version": "1.0.18",
"author": "Verdnatura Levante SL",
"description": "MySQL Version Control",
"license": "GPL-3.0",

View File

@ -1,24 +1,21 @@
CREATE DATABASE IF NOT EXISTS `util` DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
USE `util`;
CREATE TABLE `version` (
`code` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`number` char(11) COLLATE utf8_unicode_ci NOT NULL,
`gitCommit` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`code` varchar(255) NOT NULL,
`number` char(11) NOT NULL,
`gitCommit` varchar(255) NOT NULL,
`updated` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
) ENGINE=InnoDB;
ALTER TABLE `version`
ADD PRIMARY KEY (`code`);
CREATE TABLE `versionUser` (
`code` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`user` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`number` char(11) COLLATE utf8_unicode_ci NOT NULL,
`gitCommit` varchar(255) COLLATE utf8_unicode_ci NOT NULL,
`code` varchar(255) NOT NULL,
`user` varchar(255) NOT NULL,
`number` char(11) NOT NULL,
`gitCommit` varchar(255) NOT NULL,
`updated` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
) ENGINE=InnoDB;
ALTER TABLE `versionUser`
ADD PRIMARY KEY (`code`,`user`);

4
workspace/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
.DS_Store
node_modules
remotes/*.ini
.procs-priv.sql

1
workspace/dump/.dump.sql Normal file
View File

@ -0,0 +1 @@
-- Database dump will be created here

View File

@ -0,0 +1 @@
-- Place your local fixtures here

View File

@ -0,0 +1 @@
-- Place your local structure changes here

View File

@ -1,15 +1,15 @@
{
"code": "my-app",
"code": "my-db",
"schemas": [
"util",
"my_app"
"myvc",
"my_db"
],
"fixtures": {
"util": [
"myvc": [
"version",
"versionUser"
],
"my_app": [
"my_db": [
"table1",
"table2"
]

16
workspace/package.json Normal file
View File

@ -0,0 +1,16 @@
{
"name": "my-db",
"version": "1.0.0",
"author": "Me",
"description": "My database project",
"license": "GPL-3.0",
"repository": {
"type": "git"
},
"devDependencies": {
"myvc": "^1.0.17"
},
"scripts": {
"myvc": "myvc"
}
}

View File

@ -0,0 +1 @@
Routines will be automatically created here.

View File

@ -0,0 +1 @@
SET @test = NULL;

View File

@ -0,0 +1 @@
Place your versions here.