fs-words.git

@@ -0,0 +1,4 @@

+#!/usr/bin/env bash

+

+make --file=tools/makefile

+

@@ -1,17 +0,0 @@

-import sys

-import json

-

-def main():

-	content = sys.stdin.read()

-	try:

-		data = json.loads(content)

-		print(content)

-		# print("valid")

-		return 0

-	except ValueError as error:

-		print("invalid")

-		print(error)

-		return -1

-

-sys.exit(main())

-

@@ -0,0 +1,4 @@

+#!/usr/bin/env bash

+

+rm -rf build

+

@@ -1,111 +0,0 @@

-import json as _json

-import os as _os

-import re as _re

-

-

-def dokuwiki_to(data):

-	content = ""

-	content += ("  * type: %s\n" % (data["type"]))

-	content += ("  * description: %s\n" % ("?" if (data["description"] is None) else data["description"]))

-	content += ("  * tags\n")

-	for tag in data["tags"]:

-		content += ("    * %s\n" % (tag))

-	content += ("  * translations\n")

-	for (language , lemmata) in data["translations"].items():

-		content += ("    * %s\n" % (language))

-		for lemma in lemmata:

-			content += ("      * %s\n" % (lemma))

-	return content

-	

-

-def dokuwiki_from(content):

-	data = {}

-	state = "type"

-	lines = content.split("\n")

-	for line in lines:

-		if (line != ""):

-			if (state == "type"):

-				result = _re.match("^  \\* type: (.*)$", line)

-				if (result is not None):

-					data["type"] = result.group(1)

-					state = "description"

-				else:

-					raise ValueError("type expected in line '%s' while being in state '%s'" % (line, state))

-			elif (state == "description"):

-				result = _re.match("^  \\* description: (.*)$", line)

-				if (result is not None):

-					data["description"] = (None if (result.group(1) == "?") else result.group(1))

-					state = "tags"

-				else:

-					raise ValueError("description expected in line '%s' while being in state '%s'" % (line, state))

-			elif (state == "tags"):

-				result = _re.match("^  \\* tags$", line)

-				if (result is not None):

-					data["tags"] = []

-					state = "tagentry"

-				else:

-					raise ValueError("tags expected in line '%s' while being in state '%s'" % (line, state))

-			elif (state == "tagentry"):

-				result = _re.match("^    \\* (.*)$", line)

-				if (result is not None):

-					data["tags"].append(result.group(1))

-					state = "tagentry"

-				else:

-					result = _re.match("^  \\* translations$", line)

-					if (result is not None):

-						data["translations"] = {}

-						state = "language"

-					else:

-						raise ValueError("tag or translations expected in line '%s' while being in state '%s'" % (line, state))

-			elif (state == "language"):

-				result = _re.match("^    \\* (.*)$", line)

-				if (result is not None):

-					language = result.group(1)

-					data["translations"][language] = []

-					state = "lemma"

-				else:

-					raise ValueError("language expected in line '%s' while being in state '%s'" % (line, state))

-			elif (state == "lemma"):

-				result = _re.match("^      \\* (.*)$", line)

-				if (result is not None):

-					lemma = result.group(1)

-					data["translations"][language].append(lemma)

-					state = "lemma"

-				else:

-					result = _re.match("^    \\* (.*)$", line)

-					if (result is not None):

-						language = result.group(1)

-						data["translations"][language] = []

-						state = "lemma"

-					else:

-						raise ValueError("language or lemma expected in line '%s' while being in state '%s'" % (line, state))

-			else:

-				raise ValueError("unhandled state '%s'" % (state))

-	return data

-	

-

-def json_to(data):

-	content = _json.dumps(data, indent = "\t", ensure_ascii = False, sort_keys = True)

-	return content

-	

-

-def json_from(content):

-	data = _json.loads(content)

-	return data

-	

-

-def filemap(folder_from, convert_from, folder_to, convert_to, namemapper):

-	for name_from in _os.listdir(folder_from):

-		path_from = _os.path.join(folder_from, name_from)

-		handle_from = open(path_from, "r")

-		content_from = handle_from.read()

-		handle_from.close()

-		data = convert_from(content_from)

-		name_to = namemapper(name_from)

-		path_to = _os.path.join(folder_to, name_to)

-		content_to = convert_to(data)

-		handle_to = open(path_to, "w")

-		handle_to.write(content_to)

-		handle_to.close()

-	

-

@@ -1,50 +0,0 @@

-#!/usr/bin/env python3

-

-import sys as _sys

-import os as _os

-import sqlite3 as _sqlite3

-import json as _json

-

-

-def main(args):

-	path = _os.path.join("queries", "convert.sql.tpl")

-	handle = open(path, "r")

-	querytemplate = handle.read()

-	handle.close()

-	arguments = {}

-	connection = _sqlite3.connect(_os.path.join("db", "words.sqlite"))

-	result = connection.cursor().execute(querytemplate, arguments).fetchall()

-	connection.close()

-	for index in range(len(result)):

-		line = result[index]

-		if True:

-			id_ = line[0]

-		if True:

-			type_ = line[1]

-		if True:

-			description = line[2]

-		if True:

-			tags = ([] if (line[3] is None) else line[3].split("|"))

-		if True:

-			translations = {}

-			pairs = list(map(lambda x: tuple(x.split(":", 2)), line[4].split("|")))

-			for (language , lemma) in sorted(pairs, key = lambda pair: pair[0]):

-				if (language not in translations):

-					translations[language] = []

-				translations[language].append(lemma)

-		obj = {

-			"type": type_,

-			"description": description,

-			"tags": tags,

-			"translations": translations,

-		}

-		name = ("concept_%u.json" % (id_))

-		path = _os.path.join("concepts", name)

-		output = _json.dumps(obj, indent = "\t", ensure_ascii = False)

-		handle = open(path, "w")

-		handle.write(output)

-		handle.close()

-	

-

-main(_sys.argv[1:])

-

@@ -1,33 +0,0 @@

-import sys

-import json

-

-def stringify(x):

-	if (x is None):

-		return "null"

-	elif (type(x) == str):

-		return ("\"%s\"" % x)

-	elif (type(x) == list):

-		return ("[%s]" % (",".join(map(lambda y: stringify(y), x))))

-	else:

-		return str(x)

-

-def main():

-	languages = ["fs","en","af","nl","yi","de","da","nb","sv","nn","is","pg"]

-	data = json.loads(sys.stdin.read())

-	sys.stdout.write("[\n")

-	first = True

-	for entry in data:

-		if (first):

-			first = False

-		else:

-			sys.stdout.write(",\n")

-		s = ""

-		s += "\t{\n"

-		s += "\t\t%s: %s, %s: %s, %s: %s,\n" % (stringify("type"), stringify(entry["type"]), stringify("description"), stringify(entry["description"]), stringify("tags"), stringify(entry["tags"]))

-		s += "\t\t%s: {%s}\n" % (stringify("translations"), ", ".join(map(lambda language: "%s: %s" % (stringify(language), stringify(entry["translations"][language])), languages)))

-		s += "\t}"

-		sys.stdout.write(s)

-	sys.stdout.write("\n]\n")

-

-main()

-

@@ -0,0 +1,47 @@

+## directories

+dir_source := source

+dir_build := build

+dir_lib := lib

+

+## commands

+cmd_dir_make := mkdir --parents

+cmd_copy := cp --recursive --update --verbose

+cmd_tsc := tsc --lib es2015,dom --target es6

+cmd_remove := rm --force

+cmd_link := ln --symbolic

+

+

+all: ${dir_build}/sql ${dir_build}/manage

+.PHONY: all

+

+${dir_build}/sql:

+	@ ${cmd_dir_make} ${dir_build}/sql

+	@ ${cmd_copy} ${dir_source}/sql/* ${dir_build}/sql/

+.PHONY: ${dir_build}/sql

+

+${dir_build}/manage: \

+	${dir_source}/types.ts \

+	${dir_source}/helpers/string.ts \

+	${dir_source}/helpers/file.ts \

+	${dir_source}/helpers/database.ts \

+	${dir_source}/helpers/misc.ts \

+	${dir_source}/helpers/module.ts \

+	${dir_source}/helpers/storage.ts \

+	${dir_source}/helpers/repository.ts \

+	${dir_source}/entities/concept.ts \

+	${dir_source}/repositories/language.ts \

+	${dir_source}/repositories/type.ts \

+	${dir_source}/repositories/tag.ts \

+	${dir_source}/repositories/concept-core.ts \

+	${dir_source}/repositories/concept-tags.ts \

+	${dir_source}/repositories/concept-translations.ts \

+	${dir_source}/repositories/concept.ts \

+	${dir_source}/services/language.ts \

+	${dir_source}/services/type.ts \

+	${dir_source}/services/tag.ts \

+	${dir_source}/services/concept.ts \

+	${dir_source}/main.ts

+	@ ${cmd_dir_make} ${dir_build}

+	@ ${cmd_tsc} $^ --outFile $@

+	@ chmod +x ${dir_build}/manage

+

@@ -1,21 +0,0 @@

-#!/usr/bin/env python3

-

-import os as _os

-import json as _json

-

-

-def main():

-	data = []

-	folder = "concepts/json"

-	for name in sorted(_os.listdir(folder)):

-		path = _os.path.join(folder, name)

-		handle = open(path, "r")

-		content = handle.read()

-		handle.close()

-		entry = _json.loads(content)

-		data.append(entry)

-	print(_json.dumps(data))

-	

-

-main()

-

@@ -1,17 +0,0 @@

-#!/usr/bin/env sh

-

-path=concepts.json

-python3 tools/merge.py > ${path}

-

-destination=mehl:/home/bfadmin/websites/folksprak.org/htdocs/data

-rsync \

-	--verbose \

-	--archive \

-	--compress \

-	--recursive \

-	--rsh=ssh \

-	${path} \

-	${destination}/

-

-rm ${path}

-

@@ -1,23 +0,0 @@

-import sys

-import json

-

-def index(type_):

-	return {

-		"verb": 00,

-		"attribute": 10,

-		"noun": 20,

-		"pronoun": 30,

-		"correlative": 40,

-		"preposition": 50,

-		"article": 60,

-		"number": 70,

-		"other": 80,

-	}[type_]

-

-def main():

-	data = json.loads(sys.stdin.read())

-	data_ = sorted(data, key = lambda entry: index(entry["type"]))

-	print(json.dumps(data_, indent="\t"))

-

-main()

-

@@ -1,35 +0,0 @@

-import sys

-import json

-

-def sqlformat(x):

-	if (x is None):

-		return "NULL"

-	elif (type(x) == str):

-		return ("'%s'" % x.replace("'", "\\'"))

-	else:

-		return str(x)

-

-def lastid(tablename):

-	return ("(SELECT id FROM %s ORDER BY id DESC LIMIT 1)" % (tablename))

-

-def main():

-	data = json.loads(sys.stdin.read())

-	print(len(data))

-	return

-	for entry in data[324:]:

-		query = ""

-		query += ("INSERT INTO\n\tconcepts(type,description)\nVALUES\n\t(%s,%s)\n;\n" % (sqlformat(entry["type"]), sqlformat(entry["description"])))

-		concept_id = lastid("concepts")

-		for (iso_639_2,lemmata) in list(entry["translations"].items()):

-			for lemma in lemmata:

-				query += ("INSERT INTO\n\twords(language_id,lemma,pronunciation)\nVALUES\n\t((SELECT id FROM languages WHERE iso_639_2=%s),%s,%s)\n;\n" % (sqlformat(iso_639_2), sqlformat(lemma), sqlformat(None)))

-				word_id = lastid("words")

-				strength = None

-				query += ("INSERT INTO\n\tcorrelations(concept_id,word_id,strength)\nVALUES\n\t(%s,%s,%s)\n;\n" % (concept_id, word_id, sqlformat(strength)))

-		for tag in entry["tags"]:

-			query += ("INSERT INTO tags(concept_id,tag) VALUES (%s,%s);\n" % (concept_id, sqlformat(tag)))

-		query += "--\n"

-		sys.stdout.write(query)

-

-main()

-

@@ -1,19 +0,0 @@

-#!/usr/bin/env python3

-

-import sys as _sys

-

-import common as _common

-

-

-def main(args):

-	_common.filemap(

-		"concepts/json",

-		_common.json_from,

-		"concepts/wiki",

-		_common.dokuwiki_to,

-		lambda name_from: ("concept_%u.txt" % (int(name_from.split(".")[0].split("_")[1])))

-	)

-	

-

-main(_sys.argv[1:])

-

@@ -1,14 +0,0 @@

-#!/usr/bin/env sh

-

-source=mehl:/home/bfadmin/websites/folksprak.org/htdocs/wiki/data/pages/concepts

-destination=concepts/wiki

-mkdir --parents ${destination}

-rsync \

-	--verbose \

-	--archive \

-	--compress \

-	--recursive \

-	--rsh=ssh \

-	${source}/* \

-	${destination}/

-

@@ -1,13 +0,0 @@

-#!/usr/bin/env sh

-

-source=concepts/wiki

-destination=mehl:/home/bfadmin/websites/folksprak.org/htdocs/wiki/data/pages/concepts

-rsync \

-	--verbose \

-	--archive \

-	--compress \

-	--recursive \

-	--rsh=ssh \

-	${source}/* \

-	${destination}/

-

@@ -1,5 +0,0 @@

-#!/usr/bin/env sh

-

-tools/wiki-in.sh

-python3 tools/wiki-take.py

-

@@ -1,5 +0,0 @@

-#!/usr/bin/env sh

-

-python3 tools/wiki-give.py

-tools/wiki-out.sh

-

@@ -1,19 +0,0 @@

-#!/usr/bin/env python3

-

-import sys as _sys

-

-import common as _common

-

-

-def main(args):

-	_common.filemap(

-		"concepts/wiki",

-		_common.dokuwiki_from,

-		"concepts/json",

-		_common.json_to,

-		lambda name_from: ("concept_%u.json" % (int(name_from.split(".")[0].split("_")[1])))

-	)

-	

-

-main(_sys.argv[1:])

-