Ar adm 52 add audio source type to cu types (#51)

* add type

* create CU fror source

* add validator

* use source uid for cu uid

* add to source link cu type source

* fix testes

* fix load cu

* remove test file

* after code review

* fetch cu translation from source

* move get i18n from source to client

* code review changes

* rename

* fix

* remove artefacts

* use same uid for source and CU

* use uid

* filter parents sources

* remove secure from create CU

* create diff object

* remove multi thread

* compere data from .tar with data from DB and with data from file storage (Shirai)

* fix test

api/rest.go

@@ -2119,6 +2119,10 @@ func handleUpdateContentUnit(cp utils.ContextProvider, exec boil.Executor, cu *P
 		return nil, NewForbiddenError()
 	}
 
+	if unit.TypeID == common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID {
+		return nil, NewBadRequestError(errors.Errorf("Unit type %s is close for changes", common.CT_SOURCE))
+	}
+
 	if cu.Secure.Valid {
 		unit.Secure = cu.Secure.Int16
 		err = unit.Update(exec, "secure")
@@ -2171,6 +2175,10 @@ func handleUpdateContentUnitI18n(cp utils.ContextProvider, exec boil.Executor, i
 		return nil, NewForbiddenError()
 	}
 
+	if unit.TypeID == common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID {
+		return nil, NewBadRequestError(errors.Errorf("Unit type %s is close for changes", common.CT_SOURCE))
+	}
+
 	// Upsert all new i18ns
 	nI18n := make(map[string]*models.ContentUnitI18n, len(i18ns))
 	for _, i18n := range i18ns {
@@ -3489,7 +3497,7 @@ func handleCreateSource(exec boil.Executor, r CreateSourceRequest) (*Source, *Ht
 	}
 
 	// save source to DB
-	uid, err := GetFreeUID(exec, new(SourceUIDChecker))
+	uid, err := getUniqSourceAndCUUID(exec, 0)
 	if err != nil {
 		return nil, NewInternalError(err)
 	}
@@ -3515,10 +3523,44 @@ func handleCreateSource(exec boil.Executor, r CreateSourceRequest) (*Source, *Ht
 			return nil, NewInternalError(err)
 		}
 	}
+	// create CU type source
+
+	props, _ := json.Marshal(map[string]string{"source_id": s.UID})
+
+	cu := &models.ContentUnit{
+		UID:        s.UID,
+		TypeID:     common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID,
+		Published:  true,
+		Properties: null.JSONFrom(props),
+	}
+
+	err = cu.Insert(exec)
+	if err != nil {
+		return nil, NewInternalError(err)
+	}
 
 	return handleGetSource(exec, s.ID)
 }
 
+func getUniqSourceAndCUUID(exec boil.Executor, attempts int64) (string, error) {
+	if attempts > 10 {
+		return "", errors.New("Too match attempts of find unique UID for CU and Source")
+	}
+	uid, err := GetFreeUID(exec, new(SourceUIDChecker))
+	if err != nil {
+		return "", NewInternalError(err)
+	}
+	hasCU, err := models.ContentUnits(exec, qm.Where("uid = ?", uid)).Exists()
+	if err != nil {
+		return "", NewInternalError(err)
+	}
+	if hasCU {
+		attempts++
+		return getUniqSourceAndCUUID(exec, attempts)
+	}
+	return uid, err
+}
+
 func handleGetSource(exec boil.Executor, id int64) (*Source, *HttpError) {
 	source, err := models.Sources(exec,
 		qm.Where("id = ?", id),
@@ -3538,7 +3580,6 @@ func handleGetSource(exec boil.Executor, id int64) (*Source, *HttpError) {
 	for _, i18n := range source.R.SourceI18ns {
 		x.I18n[i18n.Language] = i18n
 	}
-
 	return x, nil
 }
 

cmd/cu_source.go

@@ -0,0 +1,31 @@
+package cmd
+
+import (
+	"github.com/Bnei-Baruch/mdb/importer/cusource"
+	"github.com/spf13/cobra"
+)
+
+var buildCUSourcesCmd = &cobra.Command{
+	Use:   "cu_sources",
+	Short: "Build audio_sources units",
+	Run:   buildCUSourcesCmdFn,
+}
+
+var buildCUSourcesValidatorCmd = &cobra.Command{
+	Use:   "cu_sources_validator",
+	Short: "Validate script audio_sources units",
+	Run:   buildCUSourcesValidatorCmdFn,
+}
+
+func init() {
+	RootCmd.AddCommand(buildCUSourcesCmd)
+	RootCmd.AddCommand(buildCUSourcesValidatorCmd)
+}
+
+func buildCUSourcesCmdFn(cmd *cobra.Command, args []string) {
+	cusource.InitBuildCUSources()
+}
+
+func buildCUSourcesValidatorCmdFn(cmd *cobra.Command, args []string) {
+	new(cusource.ComparatorDbVsFolder).Run()
+}

common/consts.go

@@ -44,6 +44,7 @@ const (
 	CT_BLOG_POST             = "BLOG_POST"
 	CT_RESEARCH_MATERIAL     = "RESEARCH_MATERIAL"
 	CT_KTAIM_NIVCHARIM       = "KTAIM_NIVCHARIM"
+	CT_SOURCE                = "SOURCE"
 
 	// Operation Types
 	OP_CAPTURE_START = "capture_start"

common/registry.go

@@ -75,7 +75,7 @@ var (
 		CT_UNITY_DAY, CT_CLIPS, CT_ARTICLES, CT_LESSONS_SERIES, CT_SONGS, CT_BOOKS, CT_LESSON_PART, CT_LECTURE,
 		CT_CHILDREN_LESSON, CT_WOMEN_LESSON, CT_VIRTUAL_LESSON, CT_FRIENDS_GATHERING, CT_MEAL, CT_VIDEO_PROGRAM_CHAPTER,
 		CT_FULL_LESSON, CT_ARTICLE, CT_EVENT_PART, CT_UNKNOWN, CT_CLIP, CT_TRAINING, CT_KITEI_MAKOR, CT_PUBLICATION,
-		CT_LELO_MIKUD, CT_SONG, CT_BOOK, CT_BLOG_POST, CT_RESEARCH_MATERIAL, CT_KTAIM_NIVCHARIM,
+		CT_LELO_MIKUD, CT_SONG, CT_BOOK, CT_BLOG_POST, CT_RESEARCH_MATERIAL, CT_KTAIM_NIVCHARIM, CT_SOURCE,
 	}
 
 	ALL_OPERATION_TYPES = []string{

common/utils.go

@@ -20,4 +20,3 @@ func StdLang(lang string) string {
 
 	return LANG_UNKNOWN
 }
-

config.sample.toml

@@ -51,3 +51,10 @@ consumer-secret=""
 url="https://www.laitman.ru/"
 username=""
 password=""
+
+
+[source-import]
+source-dir = "/home/david/Downloads/sources.tar.gz"
+test-url = "postgres://mdb_dev:[email protected]/mdb_dev_2?sslmode=disable"
+output = "/home/david/Downloads/validation-results.csv"
+dont-remove-dir = false

data/i18n.json

@@ -439,6 +439,17 @@
     "cs": "Vybraná zvýraznění",
     "tr": "Seçilen Önemli Noktalar"
   },
+  "content_type.SOURCE": {
+    "he": "Source unit",
+    "en": "Source unit",
+    "ru": "Source unit",
+    "es": "Source unit",
+    "ua": "Source unit",
+    "de": "Source unit",
+    "it": "Source unit",
+    "cs": "Source unit",
+    "tr": "Source unit"
+  },
   "lang.en": {
     "es": "Ingles",
     "he": "אנגלית",

importer/cusource/build_cu.go

@@ -0,0 +1,74 @@
+package cusource
+
+import (
+	"database/sql"
+	"encoding/json"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/volatiletech/sqlboiler/boil"
+	"github.com/volatiletech/sqlboiler/queries"
+	"github.com/volatiletech/sqlboiler/queries/qm"
+	"gopkg.in/volatiletech/null.v6"
+
+	"github.com/Bnei-Baruch/mdb/common"
+	"github.com/Bnei-Baruch/mdb/models"
+	"github.com/Bnei-Baruch/mdb/utils"
+)
+
+func BuildCUSources(mdb *sql.DB) ([]*models.Source, []*models.ContentUnit) {
+
+	rows, err := queries.Raw(mdb,
+		`SELECT cu.properties->>'source_id' FROM content_units cu WHERE cu.type_id = $1`,
+		common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID,
+	).Query()
+
+	utils.Must(err)
+	defer rows.Close()
+	ids := make([]int64, 0)
+	for rows.Next() {
+		var id int64
+		err := rows.Scan(&id)
+		utils.Must(err)
+		ids = append(ids, id)
+	}
+	mods := make([]qm.QueryMod, 0)
+	if len(ids) > 0 {
+		mods = append(mods, qm.WhereIn("uid NOT IN ?", utils.ConvertArgsInt64(ids)...))
+	}
+
+	sources, err := models.Sources(mdb, mods...).All()
+	utils.Must(err)
+
+	for _, s := range sources {
+		isParent := false
+		for _, sl := range sources {
+			if sl.ParentID.Int64 == s.ID {
+				isParent = true
+			}
+		}
+		if isParent {
+			continue
+		}
+		_, err := createCU(s, mdb)
+		if err != nil {
+			log.Debug("Duplicate create CU", err)
+		}
+	}
+	return sources, nil
+}
+
+func createCU(s *models.Source, mdb boil.Executor) (*models.ContentUnit, error) {
+	props, _ := json.Marshal(map[string]string{"source_id": s.UID})
+	cu := &models.ContentUnit{
+		UID:        s.UID,
+		TypeID:     common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID,
+		Published:  true,
+		Properties: null.JSONFrom(props),
+	}
+
+	err := cu.Insert(mdb)
+	if err != nil {
+		return nil, err
+	}
+	return cu, nil
+}

importer/cusource/cu_source.go

@@ -0,0 +1,22 @@
+package cusource
+
+import (
+	"database/sql"
+
+	"github.com/Bnei-Baruch/mdb/common"
+	"github.com/Bnei-Baruch/mdb/utils"
+	"github.com/spf13/viper"
+	"github.com/volatiletech/sqlboiler/boil"
+)
+
+func InitBuildCUSources() {
+	mdb, err := sql.Open("postgres", viper.GetString("mdb.url"))
+	defer mdb.Close()
+	utils.Must(err)
+	utils.Must(mdb.Ping())
+	boil.SetDB(mdb)
+	boil.DebugMode = true
+	utils.Must(common.InitTypeRegistries(mdb))
+
+	BuildCUSources(mdb)
+}