diff --git a/api/rest.go b/api/rest.go index 827ebf54..c72d67c8 100644 --- a/api/rest.go +++ b/api/rest.go @@ -2119,6 +2119,10 @@ func handleUpdateContentUnit(cp utils.ContextProvider, exec boil.Executor, cu *P return nil, NewForbiddenError() } + if unit.TypeID == common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID { + return nil, NewBadRequestError(errors.Errorf("Unit type %s is close for changes", common.CT_SOURCE)) + } + if cu.Secure.Valid { unit.Secure = cu.Secure.Int16 err = unit.Update(exec, "secure") @@ -2171,6 +2175,10 @@ func handleUpdateContentUnitI18n(cp utils.ContextProvider, exec boil.Executor, i return nil, NewForbiddenError() } + if unit.TypeID == common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID { + return nil, NewBadRequestError(errors.Errorf("Unit type %s is close for changes", common.CT_SOURCE)) + } + // Upsert all new i18ns nI18n := make(map[string]*models.ContentUnitI18n, len(i18ns)) for _, i18n := range i18ns { @@ -3489,7 +3497,7 @@ func handleCreateSource(exec boil.Executor, r CreateSourceRequest) (*Source, *Ht } // save source to DB - uid, err := GetFreeUID(exec, new(SourceUIDChecker)) + uid, err := getUniqSourceAndCUUID(exec, 0) if err != nil { return nil, NewInternalError(err) } @@ -3515,10 +3523,44 @@ func handleCreateSource(exec boil.Executor, r CreateSourceRequest) (*Source, *Ht return nil, NewInternalError(err) } } + // create CU type source + + props, _ := json.Marshal(map[string]string{"source_id": s.UID}) + + cu := &models.ContentUnit{ + UID: s.UID, + TypeID: common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID, + Published: true, + Properties: null.JSONFrom(props), + } + + err = cu.Insert(exec) + if err != nil { + return nil, NewInternalError(err) + } return handleGetSource(exec, s.ID) } +func getUniqSourceAndCUUID(exec boil.Executor, attempts int64) (string, error) { + if attempts > 10 { + return "", errors.New("Too match attempts of find unique UID for CU and Source") + } + uid, err := GetFreeUID(exec, new(SourceUIDChecker)) + if err != nil { + return "", NewInternalError(err) + } + hasCU, err := models.ContentUnits(exec, qm.Where("uid = ?", uid)).Exists() + if err != nil { + return "", NewInternalError(err) + } + if hasCU { + attempts++ + return getUniqSourceAndCUUID(exec, attempts) + } + return uid, err +} + func handleGetSource(exec boil.Executor, id int64) (*Source, *HttpError) { source, err := models.Sources(exec, qm.Where("id = ?", id), @@ -3538,7 +3580,6 @@ func handleGetSource(exec boil.Executor, id int64) (*Source, *HttpError) { for _, i18n := range source.R.SourceI18ns { x.I18n[i18n.Language] = i18n } - return x, nil } diff --git a/cmd/cu_source.go b/cmd/cu_source.go new file mode 100644 index 00000000..729d02a5 --- /dev/null +++ b/cmd/cu_source.go @@ -0,0 +1,31 @@ +package cmd + +import ( + "github.com/Bnei-Baruch/mdb/importer/cusource" + "github.com/spf13/cobra" +) + +var buildCUSourcesCmd = &cobra.Command{ + Use: "cu_sources", + Short: "Build audio_sources units", + Run: buildCUSourcesCmdFn, +} + +var buildCUSourcesValidatorCmd = &cobra.Command{ + Use: "cu_sources_validator", + Short: "Validate script audio_sources units", + Run: buildCUSourcesValidatorCmdFn, +} + +func init() { + RootCmd.AddCommand(buildCUSourcesCmd) + RootCmd.AddCommand(buildCUSourcesValidatorCmd) +} + +func buildCUSourcesCmdFn(cmd *cobra.Command, args []string) { + cusource.InitBuildCUSources() +} + +func buildCUSourcesValidatorCmdFn(cmd *cobra.Command, args []string) { + new(cusource.ComparatorDbVsFolder).Run() +} diff --git a/common/consts.go b/common/consts.go index 392a19c6..7dc0cdca 100644 --- a/common/consts.go +++ b/common/consts.go @@ -44,6 +44,7 @@ const ( CT_BLOG_POST = "BLOG_POST" CT_RESEARCH_MATERIAL = "RESEARCH_MATERIAL" CT_KTAIM_NIVCHARIM = "KTAIM_NIVCHARIM" + CT_SOURCE = "SOURCE" // Operation Types OP_CAPTURE_START = "capture_start" diff --git a/common/registry.go b/common/registry.go index c567df1e..7baa2fcd 100644 --- a/common/registry.go +++ b/common/registry.go @@ -75,7 +75,7 @@ var ( CT_UNITY_DAY, CT_CLIPS, CT_ARTICLES, CT_LESSONS_SERIES, CT_SONGS, CT_BOOKS, CT_LESSON_PART, CT_LECTURE, CT_CHILDREN_LESSON, CT_WOMEN_LESSON, CT_VIRTUAL_LESSON, CT_FRIENDS_GATHERING, CT_MEAL, CT_VIDEO_PROGRAM_CHAPTER, CT_FULL_LESSON, CT_ARTICLE, CT_EVENT_PART, CT_UNKNOWN, CT_CLIP, CT_TRAINING, CT_KITEI_MAKOR, CT_PUBLICATION, - CT_LELO_MIKUD, CT_SONG, CT_BOOK, CT_BLOG_POST, CT_RESEARCH_MATERIAL, CT_KTAIM_NIVCHARIM, + CT_LELO_MIKUD, CT_SONG, CT_BOOK, CT_BLOG_POST, CT_RESEARCH_MATERIAL, CT_KTAIM_NIVCHARIM, CT_SOURCE, } ALL_OPERATION_TYPES = []string{ diff --git a/common/utils.go b/common/utils.go index d9b7e991..bec1e9ff 100644 --- a/common/utils.go +++ b/common/utils.go @@ -20,4 +20,3 @@ func StdLang(lang string) string { return LANG_UNKNOWN } - diff --git a/config.sample.toml b/config.sample.toml index 44d04626..8e669a30 100644 --- a/config.sample.toml +++ b/config.sample.toml @@ -51,3 +51,10 @@ consumer-secret="" url="https://www.laitman.ru/" username="" password="" + + +[source-import] +source-dir = "/home/david/Downloads/sources.tar.gz" +test-url = "postgres://mdb_dev:p5eMchzHrO@pgsql.mdb.bbdomain.org/mdb_dev_2?sslmode=disable" +output = "/home/david/Downloads/validation-results.csv" +dont-remove-dir = false \ No newline at end of file diff --git a/data/i18n.json b/data/i18n.json index 5f53fe0a..6ee112eb 100644 --- a/data/i18n.json +++ b/data/i18n.json @@ -439,6 +439,17 @@ "cs": "Vybraná zvýraznění", "tr": "Seçilen Önemli Noktalar" }, + "content_type.SOURCE": { + "he": "Source unit", + "en": "Source unit", + "ru": "Source unit", + "es": "Source unit", + "ua": "Source unit", + "de": "Source unit", + "it": "Source unit", + "cs": "Source unit", + "tr": "Source unit" + }, "lang.en": { "es": "Ingles", "he": "אנגלית", diff --git a/importer/cusource/build_cu.go b/importer/cusource/build_cu.go new file mode 100644 index 00000000..db82b548 --- /dev/null +++ b/importer/cusource/build_cu.go @@ -0,0 +1,74 @@ +package cusource + +import ( + "database/sql" + "encoding/json" + + log "github.com/Sirupsen/logrus" + "github.com/volatiletech/sqlboiler/boil" + "github.com/volatiletech/sqlboiler/queries" + "github.com/volatiletech/sqlboiler/queries/qm" + "gopkg.in/volatiletech/null.v6" + + "github.com/Bnei-Baruch/mdb/common" + "github.com/Bnei-Baruch/mdb/models" + "github.com/Bnei-Baruch/mdb/utils" +) + +func BuildCUSources(mdb *sql.DB) ([]*models.Source, []*models.ContentUnit) { + + rows, err := queries.Raw(mdb, + `SELECT cu.properties->>'source_id' FROM content_units cu WHERE cu.type_id = $1`, + common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID, + ).Query() + + utils.Must(err) + defer rows.Close() + ids := make([]int64, 0) + for rows.Next() { + var id int64 + err := rows.Scan(&id) + utils.Must(err) + ids = append(ids, id) + } + mods := make([]qm.QueryMod, 0) + if len(ids) > 0 { + mods = append(mods, qm.WhereIn("uid NOT IN ?", utils.ConvertArgsInt64(ids)...)) + } + + sources, err := models.Sources(mdb, mods...).All() + utils.Must(err) + + for _, s := range sources { + isParent := false + for _, sl := range sources { + if sl.ParentID.Int64 == s.ID { + isParent = true + } + } + if isParent { + continue + } + _, err := createCU(s, mdb) + if err != nil { + log.Debug("Duplicate create CU", err) + } + } + return sources, nil +} + +func createCU(s *models.Source, mdb boil.Executor) (*models.ContentUnit, error) { + props, _ := json.Marshal(map[string]string{"source_id": s.UID}) + cu := &models.ContentUnit{ + UID: s.UID, + TypeID: common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID, + Published: true, + Properties: null.JSONFrom(props), + } + + err := cu.Insert(mdb) + if err != nil { + return nil, err + } + return cu, nil +} diff --git a/importer/cusource/cu_source.go b/importer/cusource/cu_source.go new file mode 100644 index 00000000..5d4653d8 --- /dev/null +++ b/importer/cusource/cu_source.go @@ -0,0 +1,22 @@ +package cusource + +import ( + "database/sql" + + "github.com/Bnei-Baruch/mdb/common" + "github.com/Bnei-Baruch/mdb/utils" + "github.com/spf13/viper" + "github.com/volatiletech/sqlboiler/boil" +) + +func InitBuildCUSources() { + mdb, err := sql.Open("postgres", viper.GetString("mdb.url")) + defer mdb.Close() + utils.Must(err) + utils.Must(mdb.Ping()) + boil.SetDB(mdb) + boil.DebugMode = true + utils.Must(common.InitTypeRegistries(mdb)) + + BuildCUSources(mdb) +} diff --git a/importer/cusource/validator.go b/importer/cusource/validator.go new file mode 100644 index 00000000..fdf911d2 --- /dev/null +++ b/importer/cusource/validator.go @@ -0,0 +1,255 @@ +package cusource + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha1" + "database/sql" + "encoding/hex" + "fmt" + "github.com/Bnei-Baruch/mdb/common" + "github.com/Bnei-Baruch/mdb/models" + "github.com/Bnei-Baruch/mdb/utils" + log "github.com/Sirupsen/logrus" + "github.com/spf13/viper" + "github.com/volatiletech/sqlboiler/boil" + "github.com/volatiletech/sqlboiler/queries/qm" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" +) + +type ComparatorDbVsFolder struct { + result map[string]compare + baseDir string +} + +type compare struct { + sha1 string + sUid string + name string + isOnDb bool + isOnFolder bool + isOnFileStorage bool +} + +func (c *ComparatorDbVsFolder) Run() { + c.result = make(map[string]compare) + + dir := c.untar() + mdb := c.openDB() + c.uploadData(mdb, dir) + if !viper.GetBool("source-import.dont-remove-dir") { + utils.Must(os.RemoveAll(c.baseDir)) + } + utils.Must(mdb.Close()) + +} + +func (c *ComparatorDbVsFolder) openDB() *sql.DB { + mdb, err := sql.Open("postgres", viper.GetString("source-import.test-url")) + utils.Must(err) + utils.Must(mdb.Ping()) + boil.SetDB(mdb) + boil.DebugMode = true + utils.Must(common.InitTypeRegistries(mdb)) + return mdb +} + +func (c *ComparatorDbVsFolder) untar() string { + path := viper.GetString("source-import.source-dir") + + r, err := os.Open(path) + utils.Must(err) + gzr, err := gzip.NewReader(r) + utils.Must(err) + defer utils.Must(gzr.Close()) + + c.baseDir, err = ioutil.TempDir(filepath.Dir(path), "compere_sources") + utils.Must(err) + dir := filepath.Join(c.baseDir, "from_tar") + err = os.Mkdir(dir, 0755) + utils.Must(err) + + tr := tar.NewReader(gzr) + + for { + header, err := tr.Next() + if err != nil { + if err == io.EOF { + break + } + utils.Must(err) + } + + switch header.Typeflag { + case tar.TypeDir: + err := os.Mkdir(filepath.Join(dir, header.Name), 0755) + utils.Must(err) + case tar.TypeReg: + f, err := os.Create(filepath.Join(dir, header.Name)) + utils.Must(err) + _, err = io.Copy(f, tr) + utils.Must(err) + err = f.Close() + utils.Must(err) + default: + continue + } + } + return dir +} + +func (c *ComparatorDbVsFolder) uploadData(mdb *sql.DB, dir string) { + units, err := models.ContentUnits(mdb, + qm.Where("type_id = ?", common.CONTENT_TYPE_REGISTRY.ByName[common.CT_SOURCE].ID), + qm.Load("Files"), + //qm.Limit(10), + ).All() + utils.Must(err) + for _, u := range units { + c.unitDataFromDB(*u) + } + + dir = filepath.Join(dir, "sources") + + filesOS, err := ioutil.ReadDir(dir) + utils.Must(err) + + for _, sDir := range filesOS { + c.unitDataFromFolder(sDir, dir) + } + printResults(c.result) + fmt.Print("\n******** End process ********\n") +} + +func (c *ComparatorDbVsFolder) unitDataFromDB(unit models.ContentUnit) { + sUid := unit.UID + for _, f := range unit.R.Files { + shaDb := hex.EncodeToString(f.Sha1.Bytes) + kdb := sUid + "_" + shaDb + if _, ok := c.result[kdb]; !ok { + c.result[kdb] = compare{ + sha1: shaDb, + sUid: sUid, + name: f.Name, + isOnDb: true, + } + } else { + t := c.result[kdb] + t.isOnDb = true + c.result[kdb] = t + } + + //check if have file at Shirai + var p map[string]string + err := f.Properties.Unmarshal(&p) + if err != nil { + log.Debugf("CU id - %s, file name - %s have no Properties. error: %s", sUid, f.Name, err.Error()) + continue + } + + url := "https://files.kabbalahmedia.info/files/" + f.Name + resp, err := http.Get(url) + if err != nil { + log.Errorf("Not find file on link: %s, error: %s", url, err.Error()) + continue + } + b, err := ioutil.ReadAll(resp.Body) + + if err != nil { + log.Debugf("CU id - %s, file name - %s is not on FileStorage. error: %s", sUid, f.Name, err) + continue + } + c.saveDBFile(sUid, f.Name, b) + utils.Must(resp.Body.Close()) + + shaFS := shaFromBytes(b) + log.Infof("Read from file storage, sUid: %s, file name: %s, number of bites: %d, sha: %s", sUid, f.Name, len(b), shaFS) + + kfs := sUid + "_" + shaFS + if _, ok := c.result[kfs]; !ok { + c.result[kfs] = compare{ + sha1: shaDb, + sUid: sUid, + name: f.Name, + isOnFileStorage: true, + } + } else { + t := c.result[kfs] + t.isOnFileStorage = true + c.result[kfs] = t + } + } +} + +func (c *ComparatorDbVsFolder) unitDataFromFolder(dir os.FileInfo, path string) { + if !dir.IsDir() { + log.Errorf("Not folder %s", dir.Name()) + return + } + + sUid := dir.Name() + dPath := filepath.Join(path, dir.Name()) + files, err := ioutil.ReadDir(dPath) + utils.Must(err) + + for _, file := range files { + spl := strings.Split(file.Name(), "/") + name := spl[len(spl)-1] + if isDoc := strings.Contains(name, ".doc"); !isDoc { + continue + } + + b, err := ioutil.ReadFile(filepath.Join(dPath, file.Name())) + utils.Must(err) + sha := shaFromBytes(b) + log.Infof("Read from folder, sUid: %s, file name: %s, number of bites: %d, sha: %s", sUid, name, len(b), sha) + + key := sUid + "_" + sha + + if _, ok := c.result[key]; !ok { + c.result[key] = compare{ + sha1: sha, + sUid: sUid, + name: name, + isOnFolder: true, + } + } else { + t := c.result[key] + t.isOnFolder = true + c.result[key] = t + } + } +} + +func (c *ComparatorDbVsFolder) saveDBFile(sUid, name string, file []byte) { + path := filepath.Join(c.baseDir, "from_db", sUid) + err := os.MkdirAll(path, 0755) + utils.Must(err) + + err = ioutil.WriteFile(filepath.Join(path, name), file, 0755) + utils.Must(err) +} + +/* helpers */ +func shaFromBytes(b []byte) string { + sum := sha1.Sum(b) + var sha = make(chan []byte, 1) + sha <- sum[:] + return hex.EncodeToString(<-sha) +} + +func printResults(result map[string]compare) { + lines := []string{"sha1, sUid, name, isOnDb, isOnFileStorage, isOnFolder"} + for _, d := range result { + l := fmt.Sprintf("\n%s, %s, %s, %t, %t, %t", d.sha1, d.sUid, d.name, d.isOnDb, d.isOnFileStorage, d.isOnFolder) + lines = append(lines, l) + } + b := []byte(strings.Join(lines, ",")) + err := ioutil.WriteFile(viper.GetString("source-import.output"), b, 0644) + utils.Must(err) +} diff --git a/migrations/2021-01-20_120:14:30_add_cu_source_type.sql b/migrations/2021-01-20_120:14:30_add_cu_source_type.sql new file mode 100644 index 00000000..f294b486 --- /dev/null +++ b/migrations/2021-01-20_120:14:30_add_cu_source_type.sql @@ -0,0 +1,17 @@ +-- MDB generated migration file +-- rambler up + +WITH data(name) AS (VALUES ('SOURCE')) +INSERT +INTO content_types (name) +SELECT d.name +FROM data AS d +WHERE NOT EXISTS(SELECT ct.name + FROM content_types AS ct + WHERE ct.name = d.name); + +-- rambler down + +DELETE +FROM content_types +WHERE name IN ('SOURCE');