Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(services/dropbox): fix dropbox ci #4501

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

fix(services/dropbox): fix dropbox ci #4501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
72c50e6
fix copy & rename
zjregee Apr 17, 2024
97a3dfb
fix list
zjregee Apr 17, 2024
f246234
Merge branch 'main' into fix-dropbox-ci
zjregee Apr 17, 2024
7821e65
fix typo
zjregee Apr 17, 2024
987f115
enable ci
zjregee Apr 17, 2024
d4aa276
delete firstly when overwrite
zjregee Apr 17, 2024
b0b9922
enable overwrite test
zjregee Apr 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
0 ...rvices/dropbox/dropbox/disable_action.yml → .github/services/dropbox/dropbox/action.yml
View file
Open in desktop
File renamed without changes.
50 changes: 38 additions & 12 deletions core/src/services/dropbox/backend.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,35 +192,61 @@ impl Accessor for DropboxBackend {
}

async fn copy(&self, from: &str, to: &str, _: OpCopy) -> Result<RpCopy> {
let resp = self.core.dropbox_copy(from, to).await?;
// Check if the from path exists.
let resp = self.core.dropbox_get_metadata(from).await?;
if StatusCode::OK != resp.status() {
let err = parse_error(resp).await?;
return Err(err);
}

let status = resp.status();
// Check if the to path already exists.
// We need delete it fistly due to Dropbox doesn't support overwrite.
let resp = self.core.dropbox_get_metadata(to).await?;
if StatusCode::OK == resp.status() {
let resp = self.core.dropbox_delete(to).await?;
if StatusCode::OK != resp.status() {
let err = parse_error(resp).await?;
return Err(err);
}
}

let resp = self.core.dropbox_copy(from, to).await?;
let status = resp.status();
match status {
StatusCode::OK => Ok(RpCopy::default()),
_ => {
let err = parse_error(resp).await?;
match err.kind() {
ErrorKind::NotFound => Ok(RpCopy::default()),
_ => Err(err),
}
Err(err)
}
}
}

async fn rename(&self, from: &str, to: &str, _: OpRename) -> Result<RpRename> {
let resp = self.core.dropbox_move(from, to).await?;
// Check if the from path exists.
let resp = self.core.dropbox_get_metadata(from).await?;
if StatusCode::OK != resp.status() {
let err = parse_error(resp).await?;
return Err(err);
}

let status = resp.status();
// Check if the to path already exists.
// We need delete it fistly due to Dropbox doesn't support overwrite.
let resp = self.core.dropbox_get_metadata(to).await?;
if StatusCode::OK == resp.status() {
let resp = self.core.dropbox_delete(to).await?;
if StatusCode::OK != resp.status() {
let err = parse_error(resp).await?;
return Err(err);
}
}

let resp = self.core.dropbox_move(from, to).await?;
let status = resp.status();
match status {
StatusCode::OK => Ok(RpRename::default()),
_ => {
let err = parse_error(resp).await?;
match err.kind() {
ErrorKind::NotFound => Ok(RpRename::default()),
_ => Err(err),
}
Err(err)
}
}
}
Expand Down
15 changes: 14 additions & 1 deletion core/src/services/dropbox/lister.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,17 @@ impl oio::PageList for DropboxLister {
return result;
}

// List "dir" should only return "dir/".
if !self.path.ends_with('/') {
let mut path = self.path.clone();
path.push('/');
ctx.entries
.push_back(oio::Entry::with(path, Metadata::new(EntryMode::DIR)));

ctx.done = true;
return Ok(());
}

let bytes = response.into_body();
let decoded_response: DropboxListResponse =
serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
Expand Down Expand Up @@ -103,7 +114,9 @@ impl oio::PageList for DropboxLister {
}
}

ctx.entries.push_back(oio::Entry::with(name, meta));
// Dropbox will return name without parent path, we need contact it.
let path = format!("{}{}", self.path, name);
ctx.entries.push_back(oio::Entry::with(path, meta));
}

if decoded_response.has_more {
Expand Down
4 changes: 2 additions & 2 deletions core/tests/behavior/async_copy.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ pub async fn test_copy_file_with_ascii_name(op: Operator) -> Result<()> {

/// Copy a file with non ascii name and test contents.
pub async fn test_copy_file_with_non_ascii_name(op: Operator) -> Result<()> {
// Koofr does not support non-ascii name.(https://github.com/apache/opendal/issues/4051)
if op.info().scheme() == Scheme::Koofr {
// Koofr (https://github.com/apache/opendal/issues/4051) and Dropbox does not support non-ascii name.
if op.info().scheme() == Scheme::Koofr || op.info().scheme() == Scheme::Dropbox {
return Ok(());
}

Expand Down
10 changes: 10 additions & 0 deletions core/tests/behavior/async_list.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,11 @@ pub async fn test_list_dir_with_metakey_complete(op: Operator) -> Result<()> {

/// List prefix should return newly created file.
pub async fn test_list_prefix(op: Operator) -> Result<()> {
// Dropbox does not support list prefix.
if op.info().scheme() == Scheme::Dropbox {
return Ok(());
}

let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
let (content, _) = gen_bytes(op.info().full_capability());
Expand Down Expand Up @@ -558,6 +563,11 @@ pub async fn test_list_dir_with_recursive_no_trailing_slash(op: Operator) -> Res
}

pub async fn test_list_file_with_recursive(op: Operator) -> Result<()> {
// Dropbox does not support list file with recursive.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It doesn't make sense. All services that implement list_dir should have list prefix.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure about this, I don't see it described in the API documentation, and in my practice Dropbox doesn't implement prefix matching.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

prefix matching is simulated at

opendal/core/src/layers/complete.rs

Lines 276 to 322 in 755de58

async fn complete_list(
&self,
path: &str,
args: OpList,
) -> Result<(RpList, CompleteLister<A, A::Lister>)> {
let cap = self.meta.full_capability();
if !cap.list {
return Err(self.new_unsupported_error(Operation::List));
}
let recursive = args.recursive();
match (recursive, cap.list_with_recursive) {
// - If service can list_with_recursive, we can forward list to it directly.
(_, true) => {
let (rp, p) = self.inner.list(path, args).await?;
Ok((rp, CompleteLister::One(p)))
}
// If recursive is true but service can't list_with_recursive
(true, false) => {
// Forward path that ends with /
if path.ends_with('/') {
let p = FlatLister::new(self.inner.clone(), path);
Ok((RpList::default(), CompleteLister::Two(p)))
} else {
let parent = get_parent(path);
let p = FlatLister::new(self.inner.clone(), parent);
let p = PrefixLister::new(p, path);
Ok((RpList::default(), CompleteLister::Four(p)))
}
}
// If recursive and service doesn't support list_with_recursive, we need to handle
// list prefix by ourselves.
(false, false) => {
// Forward path that ends with /
if path.ends_with('/') {
let (rp, p) = self.inner.list(path, args).await?;
Ok((rp, CompleteLister::One(p)))
} else {
let parent = get_parent(path);
let (rp, p) = self.inner.list(parent, args).await?;
let p = PrefixLister::new(p, path);
Ok((rp, CompleteLister::Three(p)))
}
}
}
}

I have a plan to refactor them later to make those behavior more clear.

if op.info().scheme() == Scheme::Dropbox {
return Ok(());
}

let parent = uuid::Uuid::new_v4().to_string();

let paths = ["y", "yy"];
Expand Down
5 changes: 5 additions & 0 deletions core/tests/behavior/blocking_list.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,11 @@ pub fn test_blocking_list_dir_with_recursive_no_trailing_slash(op: BlockingOpera
// Walk top down should output as expected
// same as test_list_dir_with_recursive except listing 'x' instead of 'x/'
pub fn test_blocking_list_file_with_recursive(op: BlockingOperator) -> Result<()> {
// Dropbox does not support list file with recursive.
if op.info().scheme() == Scheme::Dropbox {
return Ok(());
}

let parent = uuid::Uuid::new_v4().to_string();

let paths = ["y", "yy"];
Expand Down
Loading