Skip to content

Commit

Permalink
Merge branch 'vast-util-methods'
Browse files Browse the repository at this point in the history
  • Loading branch information
pgoodman committed Jul 8, 2024
2 parents 44e350e + 0815c88 commit 74f2022
Show file tree
Hide file tree
Showing 733 changed files with 18,530 additions and 14,323 deletions.
51 changes: 51 additions & 0 deletions CITATION.cff
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# This CITATION.cff file was generated with cffinit.
# Visit https://bit.ly/cffinit to generate yours today!

cff-version: 1.2.0
title: Multiplier
message: >-
If you use this software, please cite it using the
metadata from this file.
type: software
authors:
- given-names: Peter
family-names: Goodman
email: [email protected]
affiliation: Trail of Bits
- given-names: Akshay
family-names: Kumar
email: [email protected]
affiliation: Trail of Bits
repository-code: 'https://github.com/trailofbits/multiplier'
abstract: >-
Multiplier provides precise and comprehensive code
understanding capabilities. It does so by saving build
artifacts into a database, and then making them
persistently accessible using a C++ or Python API.
Multiplier emphasizes the ability to unique identify all
entities in a build process, including individual tokens,
AST nodes, and intermediate representations. With
Multiplier, an analyst can identify code patterns of
interest over one of the representations, and then
accurately relay results back to humans in a readable
form, or to follow-on scripts via entity IDs.
Multiplier's APIs are extensive, and often provide as-good
or better-than compiler-level quality information, but
linked at a whole-program granularity. We like to say that
with its APIs, you can get everywhere from anywhere.
keywords:
- c
- c++
- clang
- compiler
- index
- analysis
- intermediate representation
- ast
- abstract syntax tree
- parsing
license: Apache-2.0
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ We like to say that with its APIs, *you can get everywhere from anywhere*.
* [List all indexed variables](docs/mx-list-variables.md)
* [Search the code with regular expressions](docs/mx-regex-query.md)
* Writeups
* [regreSSHion OpenSSH variant analysis](docs/openssh-variant-analysis.md)
* [PHP variant analysis](docs/php-variant-analysis.md)

# License
Expand Down
1 change: 1 addition & 0 deletions bin/Bootstrap/PASTA.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,7 @@ static std::set<std::pair<std::string, std::string>> kMethodBlackList{
{"Expr", "ClassifyLValue"}, // Calls `clang::Expr::ClassifyImpl`.
{"Expr", "IsBoundMemberFunction"}, // Calls `clang::Expr::ClassifyImpl`.
{"Expr", "IsModifiableLvalue"}, // Calls `clang::Expr::ClassifyImpl`.
{"Expr", "HasSideEffects"}, // Recursively goes into children, triggering asserts.

// Internally can call all sorts of stuff, e.g. `InitListExpr::isTransparent`.
{"Expr", "IsCXX11ConstantExpression"},
Expand Down
101 changes: 61 additions & 40 deletions bin/Bootstrap/VAST.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -665,7 +665,38 @@ class ValueGeneratorWrapper final : public TypeWrapper {
}
};

#define IDENTITY_WRAPPER(t) {#t, new IdentityWrapper(#t)}
class TypeTypeWrapper final : public TypeWrapper {
public:
virtual ~TypeTypeWrapper(void) = default;
explicit TypeTypeWrapper(void) {}

void ReturnType(std::ostream &os, const pasta::CXXMethodDecl &) final {
os << "::mx::ir::Type";
}

virtual std::string_view MethodRefKind(void) final {
return "";
}

std::string_view CallMethod(std::ostream &os, const pasta::CXXMethodDecl &m,
const std::string &indent) {
os << indent << "auto mlir_type = underlying_repr()." << m.Name() << "();\n";
return "mlir_type";
}

void Implementation(std::ostream &os, const pasta::CXXMethodDecl &m,
const std::string &indent, std::string_view val) final {
os << indent << "return ::mx::ir::Type(\n"
<< indent << " " << val << ".getContext(),\n"
<< indent << " reinterpret_cast<const mlir::TypeStorage *>(\n"
<< indent << " " << val << ".getAsOpaquePointer()));\n";
}
};

#define IDENTITY_WRAPPER(t) \
{#t, new IdentityWrapper(#t)}, \
{"std::optional<" #t ">", new OptionalTypeWrapper(new IdentityWrapper(#t))}, \
{"::std::optional<" #t ">", new OptionalTypeWrapper(new IdentityWrapper(#t))}

static std::unordered_map<std::string, TypeWrapper *> gReturnType{
IDENTITY_WRAPPER(bool),
Expand Down Expand Up @@ -693,51 +724,21 @@ static std::unordered_map<std::string, TypeWrapper *> gReturnType{
IDENTITY_WRAPPER(std::string),
IDENTITY_WRAPPER(std::string_view),
{"::llvm::StringRef", new StringRefWrapper},
{"std::optional<StringRef>", new OptionalTypeWrapper(new StringRefWrapper)},
{"::std::optional<StringRef>", new OptionalTypeWrapper(new StringRefWrapper)},
{"::std::optional<::llvm::StringRef>", new OptionalTypeWrapper(new StringRefWrapper)},
{"::mlir::Value", new ValueWrapper},
{"::mlir::Region&", new RegionWrapper},
{"::mlir::Region &", new RegionWrapper},
{"::mlir::Block&", new BlockWrapper},
{"::mlir::Block &", new BlockWrapper},
{"::mlir::Type", new TypeTypeWrapper},
{"::mlir::mlir::Operation::result_range", new ValueGeneratorWrapper("::mx::ir::Result")},
{"::mlir::mlir::Operation::operand_range", new ValueGeneratorWrapper("::mx::ir::Operand")},
{"::std::optional<::llvm::StringRef>", new OptionalTypeWrapper(new StringRefWrapper)},
{"std::optional<StringRef>", new OptionalTypeWrapper(new StringRefWrapper)},
// {"::mlir::Attribute", "::mx::ir::Attribute"},
// {"::mlir::Value", "::mx::ir::Value"},
// {"::mlir::Type", "::mx::ir::Type"},
// {"::llvm::StringRef", "std::string_view"},
// {"std::string_view", "std::string_view"},
// {"std::string", "std::string_view"},
// {"::mlir::Region&", "::mx::ir::Region"},
// {"::std::optional<::mlir::Attribute>", "std::optional<::mx::ir::Attribute>"},
// {"::std::optional<::mlir::Value>", "std::optional<::mx::ir::Value>"},
// {"::std::optional<::mlir::Type>", "std::optional<::mx::ir::Type>"},
// {"::std::optional<::llvm::StringRef>", "std::optional<std::string_view>"},
// {"::std::optional<::mlir::Region>", "std::optional<::mx::ir::Region>"},
// {"bool", "bool"},
// {"short", "short"},
// {"int", "int"},
// {"unsigned", "unsigned"},
// {"uint8_t", "uint8_t"},
// {"uint16_t", "uint16_t"},
// {"uint32_t", "uint32_t"},
// {"int64_t", "uint64_t"},
// {"int8_t", "int8_t"},
// {"int16_t", "int16_t"},
// {"int32_t", "int32_t"},
// {"int64_t", "int64_t"},
// {"::std::optional<unsigned>", "std::optional<unsigned>"},
// {"::std::optional<uint8_t>", "std::optional<uint8_t>"},
// {"::std::optional<uint16_t>", "std::optional<uint16_t>"},
// {"::std::optional<uint32_t>", "std::optional<uint32_t>"},
// {"::std::optional<int64_t>", "std::optional<uint64_t>"},
// {"::std::optional<int8_t>", "std::optional<int8_t>"},
// {"::std::optional<int16_t>", "std::optional<int16_t>"},
// {"::std::optional<int32_t>", "std::optional<int32_t>"},
// {"::std::optional<int64_t>", "std::optional<int64_t>"},
};

void CodeGenerator::Summarize(void) {
for (Dialect &dialect : gDialects) {

for (Op &op : ops) {
if (op.root_ns == dialect.root_ns && op.ns == dialect.ns) {
dialect.ops.push_back(&op);
Expand Down Expand Up @@ -1030,6 +1031,16 @@ void CodeGenerator::RunOnOps(void) {
<< "};\n"
<< "static_assert(sizeof(Operation) == sizeof(::mx::ir::Operation));\n\n";

if (dialect.our_ns_name == "hl") {
hpp
<< "class MX_EXPORT RefOp : public Operation {\n"
<< " public:\n"
<< " static std::optional<RefOp> from(const ::mx::ir::Operation &);\n"
<< " std::optional<Symbol> referenced_symbol(void) const noexcept;\n"
<< "};\n"
<< "static_assert(sizeof(RefOp) == sizeof(Operation));\n\n";
}

cpp
<< "// Copyright (c) 2023-present, Trail of Bits, Inc.\n"
<< "// All rights reserved.\n"
Expand Down Expand Up @@ -1070,8 +1081,18 @@ void CodeGenerator::RunOnOps(void) {
std::string snake_name = OpNameToSnakeCase(op->op_name);
std::string enum_name = OpNameToEnumCase(op->op_name);

auto base = "Operation";
if (dialect.our_ns_name == "hl") {
if (op_name == "GlobalRefOp" ||
op_name == "EnumRefOp" ||
op_name == "FuncRefOp" ||
op_name == "DeclRefOp") {
base = "RefOp";
}
}

hpp
<< "class MX_EXPORT " << op_name << " final : public Operation {\n"
<< "class MX_EXPORT " << op_name << " final : public " << base << " {\n"
<< " public:\n"
<< " inline static constexpr OperationKind static_kind(void) {\n"
<< " return OperationKind::" << enum_name
Expand All @@ -1082,7 +1103,6 @@ void CodeGenerator::RunOnOps(void) {
<< " ::" << dialect.ns_key
<< "::" << op->name << " underlying_repr(void) const noexcept;\n\n"
<< " // Imported methods:\n";

cpp
<< "std::optional<" << op_name << "> " << op_name
<< "::from(const ::mx::ir::Operation &that) {\n"
Expand Down Expand Up @@ -1628,7 +1648,8 @@ void CodeGenerator::RunOnAttrs(void) {
<< "// the LICENSE file found in the root directory of this source tree.\n\n"
<< "// Auto-generated file; do not modify!\n\n"
<< "#include <multiplier/IR/" << dialect.our_dir_name.generic_string()
<< "/Attribute.h>\n\n";
<< "/Attribute.h>\n\n"
<< "#include <multiplier/IR/Type.h>\n\n";

std::set<std::string> seen;
for (Attr *attr : dialect.attrs) {
Expand Down
1 change: 1 addition & 0 deletions bin/Examples/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ define_example("mx-list-files" "ListFiles.cpp")
define_example("mx-dump-files" "DumpFiles.cpp")
define_example("mx-print-file" "PrintFile.cpp")
define_example("mx-highlight-entity" "HighlightEntity.cpp")
define_example("mx-highlight-references" "HighlightReferences.cpp")
define_example("mx-print-reference-hierarchy" "PrintReferenceHierarchy.cpp")
define_example("mx-print-call-graph" "PrintCallGraph.cpp")
define_example("mx-print-include-graph" "PrintIncludeGraph.cpp")
Expand Down
99 changes: 99 additions & 0 deletions bin/Examples/HighlightReferences.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
// Copyright (c) 2022-present, Trail of Bits, Inc.
// All rights reserved.
//
// This source code is licensed in accordance with the terms specified in
// the LICENSE file found in the root directory of this source tree.

#include <gflags/gflags.h>
#include <glog/logging.h>
#include <sstream>

#include "Index.h"
#include <multiplier/AST.h>

DEFINE_uint64(entity_id, 0, "ID of the entity to print");

namespace {

mx::TokenRange TryGetTokens(const mx::VariantEntity &entity) {
if (std::holds_alternative<mx::Stmt>(entity)) {
return std::get<mx::Stmt>(entity).tokens();
}

if (std::holds_alternative<mx::Decl>(entity)) {
return std::get<mx::Decl>(entity).tokens();
}

if (std::holds_alternative<mx::Attr>(entity)) {
return std::get<mx::Attr>(entity).tokens();
}

if (std::holds_alternative<mx::Designator>(entity)) {
return std::get<mx::Designator>(entity).tokens();
}

if (std::holds_alternative<mx::CXXBaseSpecifier>(entity)) {
return std::get<mx::CXXBaseSpecifier>(entity).tokens();
}

if (std::holds_alternative<mx::CXXCtorInitializer>(entity)) {
return std::get<mx::CXXCtorInitializer>(entity).tokens();
}

if (std::holds_alternative<mx::Macro>(entity)) {
return std::get<mx::Macro>(entity).root().use_tokens();
}

if (std::holds_alternative<mx::Token>(entity)) {
return std::get<mx::Token>(entity);
}

return mx::TokenRange();
}

} // namespace

int main(int argc, char *argv[]) {
std::stringstream ss;
ss
<< "Usage: " << argv[0]
<< " [--db DATABASE] --entity_id ID\n";

google::SetUsageMessage(ss.str());
google::ParseCommandLineFlags(&argc, &argv, false);
google::InitGoogleLogging(argv[0]);

mx::Index index = InitExample(true);

auto maybe_entity = index.entity(FLAGS_entity_id);

auto references = mx::Reference::to(maybe_entity);
for (mx::Reference ref : references) {
auto context = ref.context();
auto referer = ref.as_variant();
auto frag = mx::Fragment::containing(referer);
if (!frag) {
continue;
}

mx::TokenRange entity_tokens = TryGetTokens(context);
if (!entity_tokens) {
entity_tokens = TryGetTokens(referer);
}

if (!entity_tokens) {
continue;
}

std::cout
<< "Context ID: " << mx::EntityId(context).Pack() << '\n'
<< "Referer ID: " << mx::EntityId(referer).Pack() << '\n'
<< "Reference kind: " << ref.kind().data() << '\n';

// Print out the tokens of this fragment as they appear in the file.
RenderFragment(std::cout, *frag, entity_tokens, "", true);
std::cout << "\n\n";

}
return EXIT_SUCCESS;
}
15 changes: 0 additions & 15 deletions bin/Index/Codegen.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -294,21 +294,6 @@ class MXPreprocessingVisitorProxy : public vast::cg::fallthrough_list_node {
return next->visit(compress(type), scope);
}

vast::operation visit(const vast::cg::clang_decl *decl,
vast::cg::scope_context &scope) override {

// NOTE(pag): Workaround until VAST fixes Issue #625.
//
// XREF: https://github.com/trailofbits/vast/issues/625
if (auto func = clang::dyn_cast<clang::FunctionDecl>(decl)) {
if (auto def = func->getDefinition()) {
return next->visit(def, scope);
}
}

return next->visit(decl, scope);
}

private:

vast::cg::clang_qual_type compress(const vast::cg::clang_type *type) {
Expand Down
Loading

0 comments on commit 74f2022

Please sign in to comment.