Add a missing dependency for vast. Tweak how provenance calculates wh…

…ether or not it has changed anything. (#517) * Add a missing dependency for vast. Tweak how provenance calculates whether or not it has changed anything. * Address PR feedback * Forgot depth numbers were high positive numbers * updated pasta bootstrap files updated python bindings * Update ci.yml * update pasta commit id --------- Co-authored-by: Akshay K <[email protected]>
trailofbits · Jan 30, 2024 · 1dc8ea9 · 1dc8ea9
1 parent c9b0800
commit 1dc8ea9
Show file tree

Hide file tree

Showing 163 changed files with 13,868 additions and 13,359 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -15,7 +15,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  LLVM_PASTA_VER: ba5b66d
+  LLVM_PASTA_VER: 4a2f2bf
   LLVM_VER: 17
   PYTHON_VER: 3.11
 

diff --git a/bin/Index/PASTA.cpp b/bin/Index/PASTA.cpp
@@ -1464,6 +1464,7 @@ BuiltinTypeKind FromPasta(pasta::BuiltinTypeKind e) {
     case 465: return BuiltinTypeKind::OMP_ARRAY_SECTION;
     case 466: return BuiltinTypeKind::OMP_ARRAY_SHAPING;
     case 467: return BuiltinTypeKind::OMP_ITERATOR;
+    case 468: return BuiltinTypeKind::UNRESOLVED;
     default: __builtin_unreachable();
   }
 }

diff --git a/bin/Index/Provenance.cpp b/bin/Index/Provenance.cpp
@@ -43,6 +43,15 @@
 namespace indexer {
 namespace {
 
+static bool Update(mx::RawEntityId *val, mx::RawEntityId new_val,
+                   bool changed) {
+  if (*val != new_val) {
+    *val = new_val;
+    return true;
+  }
+  return changed;
+}
+
 static bool IsDefinableToken(pasta::TokenKind kind) {
   auto clang_kind = static_cast<clang::tok::TokenKind>(kind);
   switch (clang_kind) {
@@ -1240,35 +1249,37 @@ bool TokenProvenanceCalculator::Pull(void) {
     if (matching_parsed) {
       if (!parent_has_rel ||
           parent_related_entity_id == matching_parsed->related_entity_id) {
-        tok->parsed_token_id = matching_parsed->parsed_token_id;
-        tok->related_entity_id = matching_parsed->related_entity_id;
-        changed = true;
+        changed = Update(&(tok->parsed_token_id),
+                         matching_parsed->parsed_token_id, changed);
+        changed = Update(&(tok->related_entity_id),
+                         matching_parsed->related_entity_id, changed);
         continue;
       }
     }
 
     if (matching_rel) {
       if (!parent_has_rel) {
-        tok->related_entity_id = matching_rel->related_entity_id;
-        changed = true;
+        changed = Update(&(tok->related_entity_id),
+                         matching_rel->related_entity_id, changed);
         continue;
       }
     }
 
     if (other_parsed) {
       if (!parent_has_rel ||
           parent_related_entity_id == other_parsed->related_entity_id) {
-        tok->parsed_token_id = other_parsed->parsed_token_id;
-        tok->related_entity_id = other_parsed->related_entity_id;
-        changed = true;
+        changed = Update(&(tok->parsed_token_id),
+                         other_parsed->parsed_token_id, changed);
+        changed = Update(&(tok->related_entity_id),
+                         other_parsed->related_entity_id, changed);
         continue;
       }
     }
 
     if (other_rel) {
       if (!parent_has_rel) {
-        tok->related_entity_id = other_rel->related_entity_id;
-        changed = true;
+        changed = Update(&(tok->related_entity_id),
+                         other_rel->related_entity_id, changed);
         continue;
       }
     }
@@ -1369,7 +1380,7 @@ bool TokenProvenanceCalculator::Pull(const std::vector<TokenTreeNode> &tokens) {
 
     if (ml) {
       info->related_entity_id =
-          RelatedEntityIdToMacroToken(em, ml.value(), true  /* true */);
+          RelatedEntityIdToMacroToken(em, ml.value(), true  /* force */);
     }
   }
 
@@ -1397,15 +1408,13 @@ bool TokenProvenanceCalculator::Push(void) {
       if (derived_parsed_id == mx::kInvalidEntityId &&
           derived_rel_id == mx::kInvalidEntityId &&
           rel_id != mx::kInvalidEntityId) {
-        derived_tok->related_entity_id = rel_id;
-        derived_tok->parsed_token_id = parsed_id;
-        changed = true;
+        changed = Update(&(derived_tok->related_entity_id), rel_id, changed);
+        changed = Update(&(derived_tok->parsed_token_id), parsed_id, changed);
 
       } else if (derived_parsed_id == mx::kInvalidEntityId &&
                  derived_rel_id == rel_id &&
                  parsed_id != mx::kInvalidEntityId) {
-        derived_tok->parsed_token_id = parsed_id;
-        changed = true;
+        changed = Update(&(derived_tok->parsed_token_id), parsed_id, changed);
       }
     }
   }
@@ -1524,10 +1533,19 @@ void TokenProvenanceCalculator::Run(
     }
   }
 
+  // NOTE(pag): Depth values are actually very very large positive numbers.
+  auto min_depth = ~0u;
+  for (auto t : ordered_tokens) {
+    min_depth = std::max(min_depth, t->Depth(*this));
+  }
+
   Sort();
+
+  auto max_depth = (~0u - min_depth) + 1u;
+  auto iter = 0u;
 
   // Iteratively improve connections.
-  for (auto changed = Pull(tokens); changed; ) {
+  for (auto changed = Pull(tokens); changed && iter <= max_depth; ++iter) {
     changed = Pull();
     changed = Push() || changed;
   }

diff --git a/bin/Index/Provenance.h b/bin/Index/Provenance.h
@@ -54,6 +54,8 @@ class TokenProvenanceCalculator {
     // How many children does this node have, OR the address of the only child.
     uintptr_t child{0u};
 
+    // Zero represents unintialized, then we use the maximum valued unsigned
+    // number to represent the shallowest value.
     unsigned depth{0u};
 
 #ifndef NDEBUG

diff --git a/bin/Index/References.cpp b/bin/Index/References.cpp
@@ -191,6 +191,14 @@ static ClassificationAction ClassifyParentChild(
           return {mx::BuiltinReferenceKind::TESTS_VALUE,
                   ContinuationAction::kDoneClassifying};
         }
+      } else if (parent.Kind() == pasta::StmtKind::kForStmt) {
+        // if parent is not a conditional expr, the reference kind
+        // will be a updating the ValueDecl
+        // e.g:
+        //      int j, j;
+        //      for(i=0, j=0;;) {}
+        return {mx::BuiltinReferenceKind::UPDATES_VALUE,
+                ContinuationAction::kDoneClassifying};
       } else {
         assert(false);
       }
@@ -510,11 +518,15 @@ gap::generator<pasta::Decl> DeclReferencesFrom(pasta::Stmt stmt) {
       co_yield ref;
     }
 
-    co_yield cxx_new->OperatorNew();
+    if (auto op_new = cxx_new->OperatorNew()) {
+      co_yield op_new.value();
+    }
 
   // If we have `delete x`, then mark `` as being referenced in this fragment.
   } else if (auto cxx_del = pasta::CXXDeleteExpr::From(stmt)) {
-    co_yield cxx_del->OperatorDelete();
+    if (auto op_del = cxx_del->OperatorDelete()) {
+      co_yield op_del.value();
+    }
 
   // If we have `(T *) b` then mark `T` as being referenced in this fragment.
   } else if (auto cast = pasta::CastExpr::From(stmt)) {
@@ -572,7 +584,9 @@ gap::generator<pasta::Decl> DeclReferencesFrom(pasta::Type type) {
     }
     case pasta::TypeKind::kDependentSizedArray: {
       auto &tt = reinterpret_cast<const pasta::DependentSizedArrayType &>(type);
-      GEN(tt.SizeExpression());
+      if (auto sz_expr = tt.SizeExpression()) {
+        GEN(sz_expr.value());
+      }
       GEN(tt.ElementType());
       break;
     }
@@ -824,7 +838,9 @@ gap::generator<pasta::Decl> DeclReferencesFrom(pasta::Type type) {
     }
     case pasta::TypeKind::kUnaryTransform: {
       auto &tt = reinterpret_cast<const pasta::UnaryTransformType &>(type);
-      GEN(tt.UnderlyingType());
+      if (auto uty = tt.UnderlyingType()) {
+        GEN(uty.value());
+      }
       break;
     }
     case pasta::TypeKind::kUnresolvedUsing: {