From d91205e64e766bed706d20d750a9f3c9b81b9212 Mon Sep 17 00:00:00 2001 From: kamiyadm Date: Mon, 18 Dec 2023 17:21:38 +0800 Subject: [PATCH] :feat: add some oprations for layer add some class to manage layer. Log: --- CMakeLists.txt | 10 ++ api/JSON Schema/LayerInfo.json | 17 +++ src/linglong/package/layer/Generators.hpp | 41 ++++++ src/linglong/package/layer/LayerInfo.hpp | 39 ++++++ src/linglong/package/layer/helper.hpp | 42 ++++++ src/linglong/package/layer_dir.cpp | 49 +++++++ src/linglong/package/layer_dir.h | 32 +++++ src/linglong/package/layer_file.cpp | 78 ++++++++++++ src/linglong/package/layer_file.h | 39 ++++++ src/linglong/package/layer_info.cpp | 45 +++++++ src/linglong/package/layer_info.h | 22 ++++ src/linglong/package/layer_package.cpp | 148 ++++++++++++++++++++++ src/linglong/package/layer_package.h | 32 +++++ tools/run-quicktype.sh | 7 + 14 files changed, 601 insertions(+) create mode 100644 api/JSON Schema/LayerInfo.json create mode 100644 src/linglong/package/layer/Generators.hpp create mode 100644 src/linglong/package/layer/LayerInfo.hpp create mode 100644 src/linglong/package/layer/helper.hpp create mode 100644 src/linglong/package/layer_dir.cpp create mode 100644 src/linglong/package/layer_dir.h create mode 100644 src/linglong/package/layer_file.cpp create mode 100644 src/linglong/package/layer_file.h create mode 100644 src/linglong/package/layer_info.cpp create mode 100644 src/linglong/package/layer_info.h create mode 100644 src/linglong/package/layer_package.cpp create mode 100644 src/linglong/package/layer_package.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e85c9df3..5843f9e07 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,6 +215,16 @@ pfl_add_library( ./src/linglong/package/ref.h ./src/linglong/package_manager/package_manager.cpp ./src/linglong/package_manager/package_manager.h + ./src/linglong/package/layer/LayerInfo.hpp + ./src/linglong/package/layer/Generators.hpp + ./src/linglong/package/layer_dir.h + ./src/linglong/package/layer_dir.cpp + ./src/linglong/package/layer_file.h + ./src/linglong/package/layer_file.cpp + ./src/linglong/package/layer_info.h + ./src/linglong/package/layer_info.cpp + ./src/linglong/package/layer_package.h + ./src/linglong/package/layer_package.cpp ./src/linglong/repo/ostree_repo.cpp ./src/linglong/repo/ostree_repo.h ./src/linglong/repo/repo.cpp diff --git a/api/JSON Schema/LayerInfo.json b/api/JSON Schema/LayerInfo.json new file mode 100644 index 000000000..f701418cd --- /dev/null +++ b/api/JSON Schema/LayerInfo.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Configuration file for layer.", + "type": "object", + "properties": { + "version": { + "type": "string" + }, + "info": { + "type": "object" + } + }, + "required": [ + "version", + "info" + ] +} diff --git a/src/linglong/package/layer/Generators.hpp b/src/linglong/package/layer/Generators.hpp new file mode 100644 index 000000000..89798c777 --- /dev/null +++ b/src/linglong/package/layer/Generators.hpp @@ -0,0 +1,41 @@ +// Thish file is generated by /tools/run-quicktype.sh +// DO NOT EDIT IT. + +// clang-format off + +// To parse this JSON data, first install +// +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// Generators.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include "linglong/package/layer/helper.hpp" + +#include "linglong/package/layer/LayerInfo.hpp" + +namespace linglong { +namespace package { +namespace layer { +void from_json(const json & j, LayerInfo & x); +void to_json(json & j, const LayerInfo & x); + +inline void from_json(const json & j, LayerInfo& x) { +x.info = j.at("info").get>(); +x.version = j.at("version").get(); +} + +inline void to_json(json & j, const LayerInfo & x) { +j = json::object(); +j["info"] = x.info; +j["version"] = x.version; +} +} +} +} + +// clang-format on diff --git a/src/linglong/package/layer/LayerInfo.hpp b/src/linglong/package/layer/LayerInfo.hpp new file mode 100644 index 000000000..23884dbed --- /dev/null +++ b/src/linglong/package/layer/LayerInfo.hpp @@ -0,0 +1,39 @@ +// Thish file is generated by /tools/run-quicktype.sh +// DO NOT EDIT IT. + +// clang-format off + +// To parse this JSON data, first install +// +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// LayerInfo.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include "linglong/package/layer/helper.hpp" + +namespace linglong { +namespace package { +namespace layer { +/** +* Configuration file for layer. +*/ + +using nlohmann::json; + +/** +* Configuration file for layer. +*/ +struct LayerInfo { +std::map info; +std::string version; +}; +} +} +} + +// clang-format on diff --git a/src/linglong/package/layer/helper.hpp b/src/linglong/package/layer/helper.hpp new file mode 100644 index 000000000..d7a40602f --- /dev/null +++ b/src/linglong/package/layer/helper.hpp @@ -0,0 +1,42 @@ +// Thish file is generated by /tools/run-quicktype.sh +// DO NOT EDIT IT. + +// clang-format off + +// To parse this JSON data, first install +// +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// helper.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include + +#include + +namespace linglong { +namespace package { +namespace layer { +using nlohmann::json; + +#ifndef NLOHMANN_UNTYPED_linglong_package_layer_HELPER +#define NLOHMANN_UNTYPED_linglong_package_layer_HELPER +inline json get_untyped(const json & j, const char * property) { +if (j.find(property) != j.end()) { +return j.at(property).get(); +} +return json(); +} + +inline json get_untyped(const json & j, std::string property) { +return get_untyped(j, property.data()); +} +#endif +} +} +} + +// clang-format on diff --git a/src/linglong/package/layer_dir.cpp b/src/linglong/package/layer_dir.cpp new file mode 100644 index 000000000..a64a1aa3b --- /dev/null +++ b/src/linglong/package/layer_dir.cpp @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. + * + * SPDX-License-Identifier: LGPL-3.0-or-later + */ + +#include "linglong/package/layer_dir.h" +#include "linglong/util/qserializer/json.h" + +namespace linglong::package { + +LayerDir::~LayerDir() +{ + if(cleanup) { + this->removeRecursively(); + } +} + +void LayerDir::setCleanStatus(bool status) +{ + this->cleanup = status; +} + +utils::error::Result> LayerDir::info() const +{ + const auto infoPath = QStringList { this->absolutePath(), "info.json"}.join(QDir::separator()); + auto [info, err] = util::fromJSON>(infoPath); + if (err) { + return LINGLONG_ERR(err.code(), "failed to parse info.json"); + } + + return info; +} + +utils::error::Result LayerDir::rawInfo() const +{ + const auto infoPath = QStringList { this->absolutePath(), "info.json"}.join(QDir::separator()); + QFile file(infoPath); + if (!file.open(QIODevice::ReadOnly)) { + return LINGLONG_ERR(-1, "failed to open info.json from layer dir"); + } + + QByteArray rawData = file.readAll(); + + file.close(); + return rawData; +} + +} \ No newline at end of file diff --git a/src/linglong/package/layer_dir.h b/src/linglong/package/layer_dir.h new file mode 100644 index 000000000..32f82d607 --- /dev/null +++ b/src/linglong/package/layer_dir.h @@ -0,0 +1,32 @@ +/* + * SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. + * + * SPDX-License-Identifier: LGPL-3.0-or-later + */ + +#ifndef LINGLONG_PACKAGE_LAYER_DIR_H_ +#define LINGLONG_PACKAGE_LAYER_DIR_H_ + +#include "linglong/package/info.h" +#include "linglong/utils/error/error.h" + +#include + +namespace linglong::package { + +class LayerDir : public QDir +{ +public: + using QDir::QDir; + ~LayerDir(); + utils::error::Result> info() const; + utils::error::Result rawInfo() const; + void setCleanStatus(bool status); + +private: + bool cleanup = true; +}; + +} // namespace linglong::package + +#endif /* LINGLONG_PACKAGE_LAYER_DIR_H_ */ diff --git a/src/linglong/package/layer_file.cpp b/src/linglong/package/layer_file.cpp new file mode 100644 index 000000000..fe1d9c48e --- /dev/null +++ b/src/linglong/package/layer_file.cpp @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. + * + * SPDX-License-Identifier: LGPL-3.0-or-later + */ + +#include "linglong/package/layer_file.h" +#include "linglong/package/layer_info.h" + +#include + +namespace linglong::package { + +using nlohmann::json; + +LayerFile::~LayerFile() +{ + if (this->cleanup) { + this->remove(); + } +} + +void LayerFile::setCleanStatus(bool status) +{ + this->cleanup = status; +} + +utils::error::Result LayerFile::layerFileInfo() +{ + auto ret = layerInfoSize(); + if (!ret.has_value()) { + return LINGLONG_EWRAP("failed to get layer file size ", ret.error()); + } + + auto rawData = this->read(qint64(*ret)); + + auto layerInfo = fromJson(rawData); + if (!layerInfo.has_value()) { + return LINGLONG_EWRAP("failed to get layer info", layerInfo.error()); + } + + return layerInfo; +} + +utils::error::Result LayerFile::layerInfoSize() +{ + if (!this->isOpen() && !this->open(QIODevice::ReadOnly)) { + return LINGLONG_ERR(-1, "failed to open layer file"); + } + // read from offset 0 everytime + this->seek(0); + + quint32 layerInfoSize; + this->read(reinterpret_cast(&layerInfoSize), sizeof(quint32)); + + return layerInfoSize; +} + +utils::error::Result LayerFile::layerOffset() +{ + auto size = layerInfoSize(); + if (!size.has_value()) { + return LINGLONG_EWRAP("get LayerInfo size failed", size.error()); + } + + return *size + sizeof(quint32); +} + +utils::error::Result LayerFile::saveTo(const QString &destination) +{ + if (!this->copy(destination)) { + return LINGLONG_ERR(-1, QString("failed to save layer file to %1").arg(destination)); + } + + return LINGLONG_OK; +} + +} // namespace linglong::package \ No newline at end of file diff --git a/src/linglong/package/layer_file.h b/src/linglong/package/layer_file.h new file mode 100644 index 000000000..8ec8cca13 --- /dev/null +++ b/src/linglong/package/layer_file.h @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. + * + * SPDX-License-Identifier: LGPL-3.0-or-later + */ + +#ifndef LINGLONG_PACKAGE_LAYER_FILE_H_ +#define LINGLONG_PACKAGE_LAYER_FILE_H_ + +#include "linglong/package/layer/LayerInfo.hpp" +#include "linglong/utils/error/error.h" + +#include + +namespace linglong::package { + +// layer file [LayerInfoSize | LayerInfo | compressedData] +class LayerFile : public QFile +{ +public: + using QFile::QFile; + ~LayerFile(); + utils::error::Result layerFileInfo(); + + utils::error::Result layerOffset(); + + utils::error::Result saveTo(const QString &destination); + + void setCleanStatus(bool status); + +private: + utils::error::Result layerInfoSize(); + + bool cleanup = false; +}; + +} // namespace linglong::package + +#endif /* LINGLONG_PACKAGE_LAYER_FILE_H_ */ diff --git a/src/linglong/package/layer_info.cpp b/src/linglong/package/layer_info.cpp new file mode 100644 index 000000000..01ef906c6 --- /dev/null +++ b/src/linglong/package/layer_info.cpp @@ -0,0 +1,45 @@ +/* + * SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. + * + * SPDX-License-Identifier: LGPL-3.0-or-later + */ + +#include "linglong/package/layer_info.h" + +#include "linglong/package/layer/Generators.hpp" +#include "linglong/package/layer_file.h" + +#include + +#include + +namespace linglong::package { + +using nlohmann::json; + +utils::error::Result fromJson(const QByteArray &rawData) +{ + layer::LayerInfo layerInfo; + try { + layerInfo = json::parse(rawData).get(); + } catch (std::exception &e) { + return LINGLONG_ERR(-1, "failed to parse json value, invalid data"); + } + + return layerInfo; +} + +utils::error::Result toJson(layer::LayerInfo &layerInfo) +{ + QByteArray rawData; + json jsonValue = layerInfo; + try { + rawData = QByteArray::fromStdString(jsonValue.dump()); + } catch (std::exception &e) { + return LINGLONG_ERR(-1, "failed to dump json value, invalid data"); + } + + return rawData; +} + +} // namespace linglong::package \ No newline at end of file diff --git a/src/linglong/package/layer_info.h b/src/linglong/package/layer_info.h new file mode 100644 index 000000000..42e93d950 --- /dev/null +++ b/src/linglong/package/layer_info.h @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. + * + * SPDX-License-Identifier: LGPL-3.0-or-later + */ + +#ifndef LINGLONG_PACKAGE_LAYER_INFO_H_ +#define LINGLONG_PACKAGE_LAYER_INFO_H_ + +#include "linglong/package/layer/LayerInfo.hpp" +#include "linglong/utils/error/error.h" + +namespace linglong::package { + +const char *const layerInfoVerison = "0.1"; + +utils::error::Result fromJson(const QByteArray &rawData); +utils::error::Result toJson(layer::LayerInfo &layerInfo); + +} // namespace linglong::package + +#endif /* LINGLONG_PACKAGE_LAYER_INFO_H_ */ diff --git a/src/linglong/package/layer_package.cpp b/src/linglong/package/layer_package.cpp new file mode 100644 index 000000000..cbf7883c3 --- /dev/null +++ b/src/linglong/package/layer_package.cpp @@ -0,0 +1,148 @@ +/* + * SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. + * + * SPDX-License-Identifier: LGPL-3.0-or-later + */ + +#include "linglong/package/layer_package.h" + +#include "linglong/package/layer_info.h" +#include "linglong/util/file.h" +#include "linglong/util/runner.h" + +#include + +namespace linglong::package { + +LayerPackager::LayerPackager(const QString &workDir) + : workDir( + QStringList{ workDir, QUuid::createUuid().toString(QUuid::Id128) }.join(QDir::separator())) +{ + util::ensureDir(this->workDir); +} + +LayerPackager::~LayerPackager() +{ + util::removeDir(this->workDir); +} + +utils::error::Result> LayerPackager::pack(const LayerDir &dir, + const QString &destnation) const +{ + // compress data with erofs + const auto compressedFilePath = + QStringList{ this->workDir, "tmp.erofs" }.join(QDir::separator()); + const auto layerFilePath = QStringList{ destnation, "tmp.layer" }.join(QDir::separator()); + ; + auto ret = util::Exec("mkfs.erofs", + { "-zlz4hc,9", compressedFilePath, dir.absolutePath() }, + 15 * 60 * 1000); + if (ret) { + return LINGLONG_ERR(ret.code(), "call mkfs.erofs failed" + ret.message()); + } + + // generate LayerInfo + layer::LayerInfo layerInfo; + layerInfo.version = layerInfoVerison; + + auto rawData = dir.rawInfo(); + if (!rawData.has_value()) { + return LINGLONG_EWRAP("failed to get raw info data", rawData.error()); + } + + // rawData is not checked format + layerInfo.info = nlohmann::json::parse(*rawData); + + auto layerInfoData = toJson(layerInfo); + if (!layerInfoData.has_value()) { + return LINGLONG_EWRAP("failed to convert LayerInfo to Json data", layerInfoData.error()); + } + // write data, [LayerInfoSize | LayerInfo | compressed data ] + // open temprary layer file + QFile layer(layerFilePath); + if (!layer.open(QIODevice::WriteOnly | QIODevice::Append)) { + return LINGLONG_ERR(-1, "failed to open temporary file of layer"); + } + + // write size of LayerInfo, in 4 bytes + QDataStream out(&layer); + quint32 layerInfoSize = (*layerInfoData).size(); + out.writeRawData(reinterpret_cast(&layerInfoSize), sizeof(quint32)); + + // write LayerInfo + layer.write(*layerInfoData); + QFile compressedFile(compressedFilePath); + if (!compressedFile.open(QIODevice::ReadOnly)) { + return LINGLONG_ERR(-1, "failed to open compressed data file of layer"); + } + + // write compressedFile + qint64 chunkSize = 2 * 1024 * 1024; + qint64 remainingSize = compressedFile.size(); + qint64 compressedFileOffset = 0; + + while (remainingSize > 0) { + qint64 sizeToRead = qMin(chunkSize, remainingSize); + uchar *compressedData = compressedFile.map(compressedFileOffset, sizeToRead); + if (!compressedData) { + layer.close(); + compressedFile.close(); + + return LINGLONG_ERR(-1, "mapping compressed file error"); + } + + auto ret = layer.write(reinterpret_cast(compressedData), sizeToRead); + if (ret < 0) { + compressedFile.unmap(compressedData); + layer.close(); + compressedFile.close(); + + return LINGLONG_ERR(-1, "writing data to temprary layer file failed"); + } + + compressedFile.unmap(compressedData); + + remainingSize -= sizeToRead; + compressedFileOffset += sizeToRead; + } + + layer.close(); + compressedFile.close(); + + QSharedPointer layerFile(new LayerFile(layerFilePath)); + + return layerFile; +} + +utils::error::Result> LayerPackager::unpack(LayerFile &file, + const QString &destnation) +{ + auto unpackDir = QStringList{ this->workDir, "unpack" }.join(QDir::separator()); + util::ensureDir(unpackDir); + + QFileInfo fileInfo(file); + + auto offset = file.layerOffset(); + if (!offset.has_value()) { + return LINGLONG_EWRAP("failed to get layer offset", offset.error()); + } + auto ret = + util::Exec("erofsfuse", + { QString("--offset=%1").arg(*offset), fileInfo.absoluteFilePath(), unpackDir }); + if (ret) { + return LINGLONG_ERR(ret.code(), "call erofsfuse failed: " + ret.message()); + } + + util::copyDir(unpackDir, destnation); + + ret = util::Exec("umount", { unpackDir }); + if (ret) { + return LINGLONG_ERR(ret.code(), "call umount failed: " + ret.message()); + } + + QSharedPointer layerDir(new LayerDir(destnation)); + + return layerDir; +} + +} // namespace linglong::package \ No newline at end of file diff --git a/src/linglong/package/layer_package.h b/src/linglong/package/layer_package.h new file mode 100644 index 000000000..1ca536b67 --- /dev/null +++ b/src/linglong/package/layer_package.h @@ -0,0 +1,32 @@ +/* + * SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd. + * + * SPDX-License-Identifier: LGPL-3.0-or-later + */ + +#ifndef LINGLONG_PACKAGE_LAYER_PACKAGE_H_ +#define LINGLONG_PACKAGE_LAYER_PACKAGE_H_ + +#include "linglong/package/layer_file.h" +#include "linglong/package/layer_dir.h" +#include "linglong/utils/error/error.h" + +#include + +namespace linglong::package { + +class LayerPackager : public QObject +{ +public: + explicit LayerPackager(const QString &workDir = "/tmp/linglong-layer"); + ~LayerPackager(); + utils::error::Result> pack(const LayerDir &dir, const QString &destnation) const; + utils::error::Result> unpack(LayerFile &file, const QString &destnation); + +private: + QString workDir; +}; + +} // namespace linglong::package + +#endif /* LINGLONG_PACKAGE_LAYER_PACKAGE_H_ */ diff --git a/tools/run-quicktype.sh b/tools/run-quicktype.sh index 88602c8ce..873d321c2 100755 --- a/tools/run-quicktype.sh +++ b/tools/run-quicktype.sh @@ -119,3 +119,10 @@ generate \ "linglong::builder::config" \ "$include" \ "linglong/builder/config" + +generate \ + "$repoRoot/api/JSON Schema/LayerInfo.json" \ + LayerInfo \ + "linglong::package::layer" \ + "$include" \ + "linglong/package/layer"