Nsc/segfault doc

.dockerignore

@@ -0,0 +1,5 @@
+massif.out.*
+.vscode
+.devcontainer
+build/
+bin/

.github/workflows/pull_request.yml

@@ -0,0 +1,25 @@
+name: pull_request
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Unshallow
+        run: git fetch --prune --unshallow && git submodule update --init --recursive
+      - uses: docker/setup-buildx-action@v1
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Just build (no push)
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: false
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/release.yml

@@ -0,0 +1,29 @@
+name: release
+
+on:
+  push:
+    branches:
+      - llvmorg-10.0.0-lsif-clang
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Unshallow
+        run: git fetch --prune --unshallow && git submodule update --init --recursive
+      - uses: docker/setup-buildx-action@v1
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: sourcegraph/lsif-clang:latest
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -0,0 +1,11 @@
+massif.out.*
+vgcore.*
+valgrind*
+.clangd
+.cache
+bin
+build
+cmake-build-debug*
+.clangd
+compile_commands.json
+dump.lsif

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/indexer/backward-cpp"]
+    path = src/indexer/backward-cpp
+    url = https://github.com/bombela/backward-cpp.git

CMakeLists.txt

@@ -3,6 +3,8 @@ cmake_minimum_required(VERSION 3.16)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_BUILD_TYPE RelWithDebInfo)
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -DNDEBUG -g")
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src)
 set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/bin)

Dockerfile

@@ -0,0 +1,20 @@
+FROM ubuntu:20.10 as build
+
+RUN apt update && apt install -y llvm-10 clang-10 libclang-10-dev cmake
+
+WORKDIR /lsif-clang
+
+COPY . .
+
+RUN CC=clang-10 CXX=clang-10 cmake -B build && make -C build -j$(nproc)
+
+FROM ubuntu:20.10
+
+RUN apt update && apt install -y libllvm10 cmake clang-10
+
+# Might as well set this, for auto-index purposes
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY --from=build /lsif-clang/bin/lsif-clang /usr/local/bin/lsif-clang
+
+ENTRYPOINT [ "lsif-clang" ]

README.md

@@ -1,6 +1,6 @@
-# lsif-clang indexer ![Status: Development](https://img.shields.io/badge/status-development-yellow?style=flat)
+# lsif-clang indexer ![Status: Development](https://img.shields.io/badge/status-beta-yellow?style=flat)
 
-![GIF displaying usage on the linux kernel.](docs/gifs/torvalds-linux.gif)
+![GIF displaying usage on the linux kernel.](docs/images/torvalds-linux.gif)
 
 This project is a fork of [clangd](https://clangd.llvm.org/) with patches to add support for outputting [LSIF indexes](https://microsoft.github.io/language-server-protocol/specifications/lsif/0.5.0/specification/). Specifically, a fork of the `clang-tools-extra/clangd` subdirectory of the [llvm-project repo](https://github.com/llvm/llvm-project/).
 

docs/compdb.md

@@ -36,11 +36,11 @@ http_archive(
 ```
 
 Then you can run:
-```
+```bash
 bazel build \
   --aspects=@bazel_compdb//:aspects.bzl%compilation_database_aspect \
-  --output_groups=compdb_files,header_files  `# this should include any generated outputs needed for cpp compilation` \
-  $(bazel query 'kind("cc_(library|binary|test|inc_library|proto_library)", //...)')'
+  --output_groups=compdb_files,header_files \ # this should include any generated outputs needed for cpp compilation
+  $(bazel query 'kind("cc_(library|binary|test|inc_library|proto_library)", //...)')
 ```
 The bazel query might look different for your project.
 

docs/examples.md

@@ -4,21 +4,21 @@ This page provides examples of generating compile_commands.json files for differ
 
 ## [github.com/torvalds/linux](https://github.com/torvalds/linux)
 
-![GIF displaying usage on the linux kernel.](gifs/torvalds-linux.gif)
+![GIF displaying usage on the linux kernel.](images/torvalds-linux.gif)
 
 Once you've installed the kernel dependencies (you can use the table in `Documentation/process/changes.rst`), run the following commands from the repository root:
 ```sh
 make allyesconfig
-make CC=clang-10 HOSTCC=clang-10 `# replace with your clang version`
-scripts/gen_compile_commands.py
+make CC=clang-10 HOSTCC=clang-10 # replace with your clang version
+scripts/clang-tools/gen_compile_commands.py
 lsif-clang compile_commands.json
 ```
 
 ## [github.com/envoyproxy/envoy](https://github.com/envoyproxy/envoy)
 
 Use the following steps:
 ```sh
-./bazel/setup_clang.sh /usr/lib/llvm-10    `# or wherever your llvm installation lives`
+./bazel/setup_clang.sh /usr/lib/llvm-10    # or wherever your llvm installation lives
 echo 'build --config=clang' > user.bazelrc
 TEST_TMPDIR=/tmp tools/gen_compilation_database.py --include_headers --run_bazel_build
 lsif-clang compile_commands.json
@@ -28,6 +28,6 @@ lsif-clang compile_commands.json
 Install the [Ninja build tool](https://ninja-build.org/), and run the following commands from the repository root:
 ```sh
 cmake -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -Dgrpc_BUILD_TESTS=ON -G Ninja
-ninja -C build $(egrep -e '^build[^:]+.pb.(cc|h|c|cpp|inc|hpp)[: ]' build/build.ninja | awk '{print $2}')  `# generate pb`
+ninja -C build $(egrep -e '^build[^:]+.pb.(cc|h|c|cpp|inc|hpp)[: ]' build/build.ninja | awk '{print $2}')  # generate pb
 lsif-clang build/compile_commands.json
 ```

docs/install.md

@@ -5,7 +5,7 @@ This project depends on LLVM and Clang. lsif-clang itself should be built agains
 ### Ubuntu (20.04)
 
 ```sh
-apt install llvm-10 clang clang-10 libclang-10-dev cmake
+apt install llvm-10 clang clang-10 libclang-10-dev cmake binutils-dev
 ```
 
 #### Older versions of Ubuntu
@@ -19,13 +19,15 @@ don't exist in the `apt` package repository.
 ### MacOS
 
 ```sh
-brew install cmake sourcegraph/brew/llvm@10
+brew install cmake sourcegraph/brew/llvm@10 binutils
 ```
 
 > Note: lsif-clang must currently be built using LLVM 10
 
 # Installation
 
+Make sure to checkout any submodules, either with `git clone --recurse-submodules ...` or `git submodule update --init --recursive`
+
 ### Ubuntu
 
 ```sh

docs/segfault.md

@@ -0,0 +1,48 @@
+# How to troubleshoot a segfault
+
+This document outlines how to configure your system to provide enough information to debug lsif-clang in the case of a segfault.
+
+## Information to be provided
+
+Along with the information gathered by reading the section below, please also provide:
+
+1. lsif-clang version (commit/tag/etc)
+1. Output of `ldd <path to lsif-clang>`
+1. lsif-clang source (self-compiled/docker/prebuilt binary)
+    - In the case of self-compiled, it may be useful to attach the binary in the bug report
+1. One or both of the below (stacktrace/core dump)
+
+## Stacktrace
+
+The lsif-clang indexer has the ability to emit a detailed stacktrace (when possible) as shown in the screenshot below if a segfault is encountered. In the case of the stacktrace not being annotated with source fragments as shown below, we may also ask for a core dump. To shorten the feedback loop, it can be beneficial to include the core dump with the original bug report.
+
+![stacktrace in terminal](images/stacktrace.png)
+
+## Core dump
+
+If there's no stacktrace, then the next option is to configure the OS to save a core dump on segfault.
+
+As the process is a bit more involved, we will outline general guidelines for coercing the system to save a core dump.
+
+### Linux
+
+1. Run `ulimit -c unlimited`
+1. Check the core-dump handler
+    - You can find the configured handler by running `cat /proc/sys/kernel/core_pattern`
+    - The output can be either a path with format verbs or a path prefixed by `|` followed by format verbs
+    - If the latter, then the binary at that path will be invoked with the core dump and you should read the manual for the given program as to where it stores core dumps and how to proper configure it to do so
+    - Else if the former, assuming the aforementioned `ulimit` command was invoked, then it should save the file at that location with the filename based on the format verbs passed
+    - e.g `/tmp/core-%e.%p.%h.%t` vs `|/usr/lib/systemd/systemd-coredump %P %u %g %s %t %c %h`
+1. Configure the core-dump handler (if necessary)
+    - This can be done by running `sysctl -w kernel.core_pattern=<your desired config>` as root
+
+### Docker
+
+If invoking `lsif-clang` from a docker container, the core dump settings are inherited from the host OS (or VM if Mac/Windows).
+In this case, it is highly recommended to configure the core pattern to save to a file that can be copied to the host from the container.
+
+### Mac OS
+
+1. Run `ulimit -c unlimited`
+1. Make sure `/cores` directory is writeable by the current user
+1. And that should be it : )

src/index/LSIFSerialization.cpp

@@ -64,10 +64,11 @@ template <> struct DenseMapInfo<clang::clangd::SymbolLocation> {
 namespace {
 struct LSIFMeta {
   LSIFMeta(llvm::raw_ostream &OS, const clang::clangd::IndexFileOut &O)
-    : OS(OS), ProjectRoot(O.ProjectRoot), Debug(O.Debug) {}
+    : OS(OS), ProjectRoot(O.ProjectRoot), Debug(O.Debug), DebugFiles(O.DebugFiles) {}
   llvm::raw_ostream &OS;
   const std::string ProjectRoot;
   const bool Debug;
+  const bool DebugFiles;
   int IDCounter = 0;
   llvm::StringMap<int> DocumentIDs;
   llvm::DenseMap<clang::clangd::SymbolID, int> ReferenceResultIDs;
@@ -76,6 +77,9 @@ struct LSIFMeta {
 
   bool contains(const char *FileURI) {
     bool contains = FileURI && llvm::StringRef(FileURI).startswith(ProjectRoot);
+    if (!contains && DebugFiles) {
+      llvm::errs() << "Ignoring file: " << llvm::StringRef(FileURI) << "\n";
+    }
     return contains;
   }
 };
@@ -254,8 +258,13 @@ void writeHoverResult(LSIFMeta &Meta, const clang::clangd::Symbol &Sym,
 
           JSONOut.attribute("value", OHover.str());
         });
-        if (Sym.Documentation.data() && !Sym.Documentation.empty())
-          JSONOut.value(Sym.Documentation);
+        if (Sym.Documentation.data() && !Sym.Documentation.empty()) {
+          auto docString = Sym.Documentation;
+          if(!llvm::json::isUTF8(docString)) {
+            docString = llvm::StringRef(llvm::json::fixUTF8(Sym.Documentation));
+          }
+          JSONOut.value(docString);
+        }
       });
     });
   });