Upload folder using huggingface_hub

.asf.yaml

@@ -0,0 +1,91 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+github:
+  description: "Apache BookKeeper - a scalable, fault tolerant and low latency storage service optimized for append-only workloads"
+  homepage: https://bookkeeper.apache.org/
+  labels:
+    - apache
+    - bookkeeper
+    - big-data
+    - distributed-log
+    - distributed-systems
+    - wal
+  features:
+    # Enable wiki for documentation
+    wiki: false
+    # Enable issues management
+    issues: true
+    # Enable projects for project management boards
+    projects: true
+  enabled_merge_buttons:
+    # enable squash button:
+    squash:  true
+    # disable merge button:
+    merge:   false
+    # disable rebase button:
+    rebase:  false
+  protected_branches:
+    master:
+      required_status_checks:
+        # strict means "Require branches to be up to date before merging".
+        strict: false
+        # Contexts are the names of checks that must pass.
+        # See ./github/workflows/README.md for more documentation on this list.
+        contexts:
+          - BookKeeper CI checks completed
+
+      required_pull_request_reviews:
+        dismiss_stale_reviews: false
+        require_code_owner_reviews: true
+        required_approving_review_count: 1
+
+      # squash or rebase must be allowed in the repo for this setting to be set to true.
+      required_linear_history: true
+
+      required_signatures: false
+
+    # The following branch protections only ensure that force pushes are not allowed
+    branch-4.0: {}
+    branch-4.1: {}
+    branch-4.2: {}
+    branch-4.3: {}
+    branch-4.4: {}
+    branch-4.5: {}
+    branch-4.6: {}
+    branch-4.7: {}
+    branch-4.8: {}
+    branch-4.9: {}
+    branch-4.10: {}
+    branch-4.11: {}
+    branch-4.12: {}
+    branch-4.13: {}
+    branch-4.14: {}
+    branch-4.15: {}
+    branch-4.16: {}
+
+publish:
+  whoami: asf-site
+
+notifications:
+  commits:      commits@bookkeeper.apache.org
+  issues:       commits@bookkeeper.apache.org
+  pullrequests: commits@bookkeeper.apache.org
+  discussions:  dev@bookkeeper.apache.org
+  jira_options: link label

.dlc.json

@@ -0,0 +1,89 @@
+{
+  "ignorePatterns": [
+    {
+      "pattern": "^http://localhost"
+    },
+    {
+      "pattern": "^https://kubernetes.io/docs/resources-reference/v1.6/"
+    },
+    {
+      "pattern": "^https://zookeeper.apache.org/doc/"
+    },
+    {
+      "pattern": "^http://zookeeper.apache.org/doc/current/api/"
+    },
+    {
+      "pattern": "^https://github.com/apache/bookkeeper/"
+    },
+    {
+      "pattern": "^https://github.com/sijie/bookkeeper/tree/bookie_shell_refactor"
+    },
+    {
+      "pattern": "^http://pulsar.incubator.apache.org/docs/latest/getting-started/ConceptsAndArchitecture/#persistent-storage"
+    },
+    {
+      "pattern": "^https://zookeeper.apache.org/doc/current/api/org/apache/zookeeper/"
+    },
+    {
+      "pattern": "^http://pulsar.incubator.apache.org/docs/latest/getting-started/ConceptsAndArchitecture/#persistent-storage"
+    },
+    {
+      "pattern": "^https://bookkeeper.apache.org/docs/api/javadoc/"
+    },
+    {
+      "pattern": "^http://hbtc2012.hadooper.cn/subject/track1maheswara2.pdf"
+    },
+    {
+      "pattern": "^https://developer.yahoo.com/blogs/ydn/bookkeeper-durability-scale-54048.html"
+    },
+    {
+      "pattern": "^http://apache.claz.org/bookkeeper"
+    },
+    {
+      "pattern": "^https://dcos.io/"
+    },
+    {
+      "pattern": "^ https://docs.mesosphere.com"
+    },
+    {
+      "pattern": "^http://master.dcos/exhibitor"
+    },
+    {
+      "pattern": "^https://docs.mesosphere.com/"
+    },
+    {
+      "pattern": "^https://distributedlog.io"
+    },
+    {
+      "pattern": "^https://www.opengroup.org/membership/forums/platform/unix"
+    },
+    {
+      "pattern": "^https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/KerberosReq.html"
+    },
+    {
+      "pattern": "^http://daxue.qq.com/content/content/id/2492"
+    },
+    {
+      "pattern": "^https://calendar.google.com/"
+    },
+    {
+      "pattern": "^#"
+    },
+    {
+      "pattern": ".*\\{\\{.*"
+    },
+    {
+      "pattern": "^//"
+    }
+  ],
+  "timeout": "60s",
+  "retryOn429": true,
+  "retryCount": 10,
+  "fallbackRetryDelay": "1000s",
+  "aliveStatusCodes": [
+    200,
+    400,
+    401,
+    403
+  ]
+}

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+site3/website/static/img/batch_read.png filter=lfs diff=lfs merge=lfs -text
+site3/website/static/img/single_read.png filter=lfs diff=lfs merge=lfs -text

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,38 @@
+Is this a question, feature request, bug report, or bookkeeper proposal?
+
+**QUESTION**
+
+Have you checked our documentation at https://bookkeeper.apache.org/ , If you could not find an answer there, please consider asking your question in our community forum at user@bookkeeper.apache.org, as it would benefit other members of our community.
+
+
+**FEATURE REQUEST**
+
+1. Please describe the feature you are requesting.
+
+2. Indicate the importance of this issue to you (blocker, must-have, should-have, nice-to-have). Are you currently using any workarounds to address this issue?
+
+3. Provide any additional detail on your proposed use case for this feature.
+
+4. If there are some sub-tasks using -[] for each subtask and create a corresponding issue to map to the sub task:
+- [ ] [sub-task1-issue-number](example_sub_issue1_link_here): sub-task1 discription here, 
+- [ ] [sub-task2-issue-number](example_sub_issue2_link_here): sub-task2 discription here,
+- ...
+
+
+**BUG REPORT**
+
+1. Please describe the issue you observed:
+
+- What did you do?
+
+- What did you expect to see?
+
+- What did you see instead?
+
+**BP**
+
+This is the master ticket for tracking BP-<bp-number> :
+
+[a short description for this BP]
+
+Proposal PR - #<pr-number>

.github/ISSUE_TEMPLATE/bp.md

@@ -0,0 +1,19 @@
+---
+name: BookKeeper Proposal
+about: Propose a major change to BookKeeper
+title: ''
+labels: type/proposal
+assignees: ''
+
+---
+
+**BP**
+
+> Follow the instructions at https://bookkeeper.apache.org/community/bookkeeper-proposals/ to create a proposal.
+
+This is the master ticket for tracking BP-xyz :
+
+[a short description for this BP]
+
+<!-- add a proposal PR link below -->
+Proposal PR - #abc

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Create a bug report to help us improve
+title: ''
+labels: type/bug
+assignees: ''
+
+---
+
+**BUG REPORT**
+
+***Describe the bug***
+
+A clear and concise description of what the bug is.
+
+***To Reproduce***
+
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+***Expected behavior***
+
+A clear and concise description of what you expected to happen.
+
+***Screenshots***
+
+If applicable, add screenshots to help explain your problem.
+
+***Additional context***
+
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: type/feature
+assignees: ''
+
+---
+
+**FEATURE REQUEST**
+
+1. Please describe the feature you are requesting.
+
+2. Indicate the importance of this issue to you (blocker, must-have, should-have, nice-to-have).
+   Are you currently using any workarounds to address this issue?
+
+3. Provide any additional detail on your proposed use case for this feature.
+

.github/ISSUE_TEMPLATE/flaky_test.md

@@ -0,0 +1,31 @@
+---
+name: Flaky test
+about: Report a flaky test failure
+title: 'Flaky-test: test_class.test_method'
+labels: ["area/tests", "flaky-tests"]
+assignees: ''
+---
+<!--- 
+
+Instructions for reporting a flaky test using this issue template:
+
+1. Replace test_class in title and body with the short test class name WITHOUT the package name.
+2. Replace test_method in title and body with the test method that failed. Multiple methods are flaky, remove the content that refers to the test method.
+3. Replace "relevant parts of the exception stacktrace here" with the a few lines of the stack trace that shows at least the exception message and the line of test code where the stacktrace occurred.
+4. Replace "full exception stacktrace here" with the full exception stacktrace from logs. This section will be hidden by default.
+5. Remove all unused fields / content to unclutter the reported issue. Remove this comment too.
+
+-->
+test_class.test_method is flaky. It fails sporadically.
+
+```
+[relevant parts of the exception stacktrace here]
+```
+
+<!-- optionally provide the full stacktrace ->
+<details>
+<summary>Full exception stacktrace</summary>
+<code><pre>
+full exception stacktrace here
+</pre></code>
+</details>

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,14 @@
+---
+name: Question
+about: Ask the community for answering questions of BookKeeper
+labels: type/question
+assignees: ''
+
+---
+
+**QUESTION**
+
+Have you checked our documentation at https://bookkeeper.apache.org/?
+If you could not find an answer there, please consider asking your question
+in our community mailing list at user@bookkeeper.apache.org, as it would benefit
+other members of our community.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,47 @@
+Descriptions of the changes in this PR:
+
+<!-- Either this PR fixes an issue, -->
+
+Fix #xyz
+
+<!-- or this PR is one task of an issue -->
+
+Main Issue: #xyz
+
+<!-- If the PR belongs to a BP, please add the BP link here -->
+
+BP: #xyz
+
+### Motivation
+
+(Explain: why you're making that change, what is the problem you're trying to solve)
+
+### Changes
+
+(Describe: what changes you have made)
+
+> ---
+> In order to uphold a high standard for quality for code contributions, Apache BookKeeper runs various precommit
+> checks for pull requests. A pull request can only be merged when it passes precommit checks.
+>
+> ---
+> Be sure to do all the following to help us incorporate your contribution
+> quickly and easily:
+>
+> If this PR is a BookKeeper Proposal (BP):
+>
+> - [ ] Make sure the PR title is formatted like:
+>     `<BP-#>: Description of bookkeeper proposal`
+>     `e.g. BP-1: 64 bits ledger is support`
+> - [ ] Attach the master issue link in the description of this PR.
+> - [ ] Attach the google doc link if the BP is written in Google Doc.
+>
+> Otherwise:
+> 
+> - [ ] Make sure the PR title is formatted like:
+>     `<Issue #>: Description of pull request`
+>     `e.g. Issue 123: Description ...`
+> - [ ] Make sure tests pass via `mvn clean apache-rat:check install spotbugs:check`.
+> - [ ] Replace `<Issue #>` in the title with the actual Issue number.
+> 
+> ---

.github/actions/bot/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "github-action-bot",
+  "description": "Bot for github actions",
+  "version": "1.1.1",
+  "author": "GitHub",
+  "license": "Apache LICENSE 2.0",
+  "main": "dist/index.js",
+  "private": true,
+  "scripts": {
+    "build": "ncc build src/run.js"
+  },
+  "dependencies": {
+    "@actions/core": "^1.2.4",
+    "@actions/github": "^2.2.0",
+    "@actions/io": "^1.0.2"
+  },
+  "devDependencies": {
+    "@types/jest": "^25.1.4",
+    "@typescript-eslint/eslint-plugin": "^2.33.0",
+    "@typescript-eslint/parser": "^2.33.0",
+    "@zeit/ncc": "^0.22.0",
+    "eslint": "^7.0.0",
+    "eslint-config-prettier": "^6.11.0",
+    "husky": "^4.2.5",
+    "jest": "^25.1.0",
+    "npm-run-all": "^4.1.5",
+    "prettier": "^2.0.5",
+    "ts-jest": "^25.2.1",
+    "typescript": "^3.8.3"
+  }
+}

.github/actions/bot/src/run.js

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+async function run(core, context, github) {
+
+    try {
+        const owner = context.repo.owner;
+        const repo = context.repo.repo;
+        const reRunCmd = process.env.RERUN_CMD;
+        const comment = context.payload.comment.body;
+
+        if (comment !== reRunCmd) {
+            core.info("this is not a bot command");
+            return;
+        }
+
+        const {
+            data: {
+                head: {
+                    sha: prRef,
+                }
+            }
+        } = await github.rest.pulls.get({
+            owner,
+            repo,
+            pull_number: context.issue.number,
+        });
+
+        const jobs = await github.rest.checks.listForRef({
+            owner,
+            repo,
+            ref: prRef,
+            status: "completed"
+        });
+
+        jobs.data.check_runs.forEach(job => {
+            if (job.conclusion === 'failure' || job.conclusion === 'cancelled') {
+                core.info("rerun job " + job.name);
+                github.rest.checks.rerequestSuite({
+                    owner,
+                    repo,
+                    check_suite_id: job.check_suite.id
+                })
+            }
+        });
+    } catch (e) {
+        core.setFailed(e);
+    }
+
+}
+
+module.exports = ({core}, {context}, {github}) => {
+    return run(core, context, github);
+}

.github/actions/clean-disk/action.yml

@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: clean disk
+description: makes some more space available on the disk by removing files
+inputs:
+  mode:
+    description: "Use 'full' to clean as much as possible"
+    required: false
+runs:
+  using: composite
+  steps:
+    - run: |
+        if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+          directories=(/usr/local/lib/android /opt/ghc)
+          if [[ "${{ inputs.mode }}" == "full" ]]; then
+            # remove these directories only when mode is 'full'
+            directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
+          fi
+          emptydir=/tmp/empty$$/
+          mkdir $emptydir
+          echo "::group::Available diskspace"
+          time df -BM / /mnt
+          echo "::endgroup::"
+          for directory in "${directories[@]}"; do
+            echo "::group::Removing $directory"
+            # fast way to delete a lot of files on linux
+            time sudo eatmydata rsync -a --delete $emptydir ${directory}/
+            time sudo eatmydata rm -rf ${directory}
+            time df -BM / /mnt
+            echo "::endgroup::"
+          done
+          echo "::group::Cleaning apt state"
+          time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
+          time df -BM / /mnt
+          echo "::endgroup::"
+        fi
+        echo "::group::Available diskspace"
+        time df -BM / /mnt
+        echo "::endgroup::"
+      shell: bash

.github/actions/copy-test-reports/action.yml

@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Copy test reports
+description: Aggregates all test reports to ./test-reports and ./surefire-reports directories
+runs:
+  using: composite
+  steps:
+    - run: |
+        $GITHUB_WORKSPACE/dev/ci-tool move_test_reports
+      shell: bash

.github/actions/tune-runner-vm/action.yml

@@ -0,0 +1,95 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Tune Runner VM performance
+description: tunes the GitHub Runner VM operation system
+runs:
+  using: composite
+  steps:
+    - run: |
+        if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+            echo "::group::Configure and tune OS"
+            # Ensure that reverse lookups for current hostname are handled properly
+            # Add the current IP address, long hostname and short hostname record to /etc/hosts file
+            echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+
+            # The default vm.swappiness setting is 60 which has a tendency to start swapping when memory
+            # consumption is high.
+            # Set vm.swappiness=1 to avoid swapping and allow high RAM usage
+            echo 1 | sudo tee /proc/sys/vm/swappiness
+            (
+              shopt -s nullglob
+              # Set swappiness to 1 for all cgroups and sub-groups
+              for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
+                echo 1 | sudo tee $swappiness_file > /dev/null
+              done
+            ) || true
+
+            # use "madvise" Linux Transparent HugePages (THP) setting
+            # https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
+            # "madvise" is generally a better option than the default "always" setting
+            # Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
+            echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
+            echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
+            echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
+            echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
+    
+            # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
+            # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
+            # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
+            # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
+            # https://www.spinics.net/lists/linux-ide/msg52562.html
+            for i in /sys/block/sd*/queue/discard_max_bytes; do
+              echo 0 | sudo tee $i
+            done
+            # disable any background jobs that run SSD discard/trim
+            sudo systemctl disable fstrim.timer || true
+            sudo systemctl stop fstrim.timer || true
+            sudo systemctl disable fstrim.service || true
+            sudo systemctl stop fstrim.service || true
+
+            # stop php-fpm
+            sudo systemctl stop php8.0-fpm.service || true
+            sudo systemctl stop php7.4-fpm.service || true
+            # stop mono-xsp4
+            sudo systemctl disable mono-xsp4.service || true
+            sudo systemctl stop mono-xsp4.service || true
+            sudo killall mono || true
+
+            # stop Azure Linux agent to save RAM
+            sudo systemctl stop walinuxagent.service || true
+            echo '::endgroup::'
+
+            # show memory
+            echo "::group::Available Memory"
+            free -m
+            echo '::endgroup::'
+            # show disk
+            echo "::group::Available diskspace"
+            df -BM
+            echo "::endgroup::"
+            # show cggroup
+            echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
+            CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
+            sudo cgget -a $CURRENT_CGGROUP || true
+            echo '::endgroup::'
+        fi
+      shell: bash

.github/changes-filter.yaml

@@ -0,0 +1,17 @@
+# contains pattern definitions used in workflows "changes" step
+# pattern syntax: https://github.com/micromatch/picomatch
+all:
+  - '**'
+docs:
+  - 'site3/**'
+  - '.asf.yaml'
+  - '*.md'
+  - '**/*.md'
+  - '.github/changes-filter.yaml'
+  - '.github/ISSUE_TEMPLATE/**'
+  - 'src/owasp-dependency-check-suppressions.xml'
+need_owasp:
+  - 'pom.xml'
+  - '**/pom.xml'
+  - 'src/owasp-dependency-check-false-positives.xml'
+  - 'src/owasp-dependency-check-suppressions.xml'

.github/workflows/README.md

@@ -0,0 +1,60 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+
+## GitHub Workflows
+
+This directory contains all BookKeeper CI checks.
+
+### Required Workflows
+
+When adding new CI workflows, please update the [.asf.yaml](../../.asf.yaml) if the workflow is required to pass before
+a PR can be merged. Instructions on how to update the file are below.
+
+This project uses the [.asf.yaml](../../.asf.yaml) to configure which workflows are required to pass before a PR can
+be merged. In the `.asf.yaml`, the required contexts are defined in the `github.protected_branches.*.required_status_checks.contexts.[]`
+where * is any key in the `protected_branches` map.
+
+You can view the currently required status checks by running the following command:
+
+```shell
+curl -s -H 'Accept: application/vnd.github.v3+json' https://api.github.com/repos/apache/bookkeeper/branches/master | \
+jq .protection
+```
+
+These contexts get their names in one of two ways depending on how the workflow file is written in this directory. The
+following command will print out the names of each file and the associated with the check. If the `name` field is `null`,
+the context will be named by the `id`.
+
+```shell
+for f in .github/workflows/*.yaml .github/workflows/*.yml; \
+do FILE=$f yq eval -o j '.jobs | to_entries | {"file": env(FILE),"id":.[].key, "name":.[].value.name}' $f; \
+done
+```
+
+Duplicate names are allowed, and all checks with the same name will be treated the same (required or not required).
+
+When working on workflow changes, one way to find out the names of the status checks is to retrieve the names
+from the PR build run. The "check-runs" can be found by commit id. Here's an example:
+
+```shell
+curl -s "https://api.github.com/repos/apache/bookkeeper/commits/$(git rev-parse HEAD)/check-runs" | \
+  jq -r '.check_runs | .[] | .name' |sort
+```

.github/workflows/bk-ci.yml

@@ -0,0 +1,604 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: BookKeeper CI
+
+on:
+  pull_request:
+    branches:
+      - master
+      - branch-*
+  push:
+    branches:
+      - master
+      - branch-*
+  workflow_dispatch:
+
+env:
+  MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-and-license-check:
+    name: PR Validation
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Detect changed files
+        id: changes
+        uses: apache/pulsar-test-infra/paths-filter@master
+        with:
+          filters: .github/changes-filter.yaml
+          list-files: csv
+
+      - name: Check changed files
+        id: check_changes
+        run: |
+          echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
+          echo "need_owasp=${{ fromJSON(steps.changes.outputs.need_owasp) }}" >> $GITHUB_OUTPUT
+
+      - name: Cache local Maven repository
+        if: steps.check_changes.outputs.docs_only != 'true'
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+
+      - name: Set up JDK 11
+        if: steps.check_changes.outputs.docs_only != 'true'
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 11
+
+      - name: Validate pull request
+        if: steps.check_changes.outputs.docs_only != 'true'
+        run: |
+          mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests
+          mvn -T 1C -B -nsu apache-rat:check checkstyle:check spotbugs:check spotless:check package -Ddistributedlog -DskipTests
+
+      - name: Check license files
+        if: steps.check_changes.outputs.docs_only != 'true'
+        run: dev/check-all-licenses
+
+      - name: Generate Javadoc
+        if: steps.check_changes.outputs.docs_only != 'true'
+        run: mvn -B -nsu -am -pl bookkeeper-common,bookkeeper-server,:bookkeeper-stats-api,:bookkeeper-stats-providers,:codahale-metrics-provider,:prometheus-metrics-provider javadoc:aggregate -DskipTests -Pdelombok -Dchesktyle.skip -Dspotbugs.skip
+    outputs:
+      docs_only: ${{ steps.check_changes.outputs.docs_only }}
+      need_owasp: ${{ steps.check_changes.outputs.need_owasp }}
+
+  unit-tests:
+    name: ${{ matrix.step_name }}
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ matrix.timeout || 60 }}
+    needs: [ 'build-and-license-check' ]
+    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - step_name: Bookie Tests
+            module: bookkeeper-server
+            flag: bookie
+            test_args: "-Dtest='org.apache.bookkeeper.bookie.**'"
+          - step_name: Client Tests
+            module: bookkeeper-server
+            flag: client
+            test_args: "-Dtest='org.apache.bookkeeper.client.**'"
+            timeout: 75
+          - step_name: Replication Tests
+            module: bookkeeper-server
+            flag: replication
+            test_args: "-Dtest='org.apache.bookkeeper.replication.**'"
+          - step_name: Remaining Tests
+            module: bookkeeper-server
+            flag: remaining
+            test_args: "-Dtest='!org.apache.bookkeeper.client.**,!org.apache.bookkeeper.bookie.**,!org.apache.bookkeeper.replication.**,!org.apache.bookkeeper.tls.**'"
+          - step_name: TLS Tests
+            module: bookkeeper-server
+            flag: tls
+            test_args: "-Dtest='org.apache.bookkeeper.tls.**'"
+          - step_name: StreamStorage Tests
+            test_args: "-f stream/pom.xml -DstreamTests"
+            flag: stream
+          - step_name: Shell tests
+            module: tests/scripts
+            flag: shell
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 11
+
+      - name: Tune Java DNS TTL settings
+        run: |
+          sudo tee -a $JAVA_HOME/conf/security/java.security <<EOF
+          networkaddress.cache.ttl=1
+          networkaddress.cache.negative.ttl=1
+          EOF
+
+      - name: Build
+        run: |
+          projects_list=
+          if [[ ! -z "${{ matrix.module }}" ]]; then
+            projects_list="-pl ${{ matrix.module }}"
+          fi
+          mvn -q -T 1C -B -nsu $projects_list install -am -DskipTests -Dcheckstyle.skip -Dspotbugs.skip -Drat.skip -Dmaven.javadoc.skip
+
+      - name: Test - ${{ matrix.step_name }}
+        run: |
+          projects_list=
+          if [[ ! -z "${{ matrix.module }}" ]]; then
+            projects_list="-pl ${{ matrix.module }}"
+          fi
+          mvn -B -nsu $projects_list verify ${{ matrix.test_args }}
+
+      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
+        if: ${{ always() }}
+        uses: ./.github/actions/copy-test-reports
+
+      - name: Publish Test Report
+        uses: apache/pulsar-test-infra/action-junit-report@master
+        if: ${{ always() }}
+        with:
+          report_paths: 'surefire-reports/TEST-*.xml'
+          annotate_only: 'true'
+
+      - name: Upload Surefire reports
+        uses: actions/upload-artifact@v4
+        if: failure()
+        continue-on-error: true
+        with:
+          name: unit-${{ matrix.step_name }}-reports
+          path: surefire-reports
+          retention-days: 7
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps
+
+  integration-tests:
+    name: Integration Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    needs: [ 'build-and-license-check' ]
+    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Clean Disk
+        uses: ./.github/actions/clean-disk
+        with:
+          mode: full
+
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 11
+
+      - name: Tune Java DNS TTL settings
+        run: |
+          sudo tee -a $JAVA_HOME/conf/security/java.security <<EOF
+          networkaddress.cache.ttl=1
+          networkaddress.cache.negative.ttl=1
+          EOF
+
+      - name: Pick ubuntu mirror for the docker image build
+        run: |
+          # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
+          $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror
+
+      - name: Build with Maven
+        run: mvn -B -nsu clean install -Pdocker -DskipTests
+
+      - name: Run metadata driver tests
+        # Exclude jetcd-core-shaded from integration tests, as it’s a POM-only project used internally,
+        # and maven prioritizes workspace artifacts during testing.
+        run: mvn -B -nsu -f metadata-drivers/pom.xml -pl '!jetcd-core-shaded' test -DintegrationTests
+
+      - name: Run all integration tests (except backward compatibility tests)
+        run: |
+          mvn -B -nsu -f tests/pom.xml test -DintegrationTests -DredirectTestOutputToFile=false -DtestRetryCount=0
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps
+
+      - name: Upload container logs on failure
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
+        continue-on-error: true
+        with:
+          retention-days: 7
+          name: integration-tests-container-logs
+          if-no-files-found: ignore
+          path: |
+            **/docker.log
+
+      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
+        if: ${{ always() }}
+        uses: ./.github/actions/copy-test-reports
+
+      - name: Publish Test Report
+        uses: apache/pulsar-test-infra/action-junit-report@master
+        if: ${{ always() }}
+        with:
+          report_paths: 'surefire-reports/TEST-*.xml'
+          annotate_only: 'true'
+
+      - name: Upload Surefire reports
+        uses: actions/upload-artifact@v4
+        if: failure()
+        continue-on-error: true
+        with:
+          name: integration-tests-reports
+          path: surefire-reports
+          if-no-files-found: ignore
+          retention-days: 7
+
+  backward-compatibility-tests:
+    name: Backward compatibility tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    needs: [ 'build-and-license-check' ]
+    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Clean Disk
+        uses: ./.github/actions/clean-disk
+        with:
+          mode: full
+
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 8
+
+      - name: Pick ubuntu mirror for the docker image build
+        run: |
+          # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
+          $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror
+
+      - name: Build with Maven
+        run: mvn -B -nsu clean install -Pdocker -DskipTests
+
+      - name: Test current server with old clients
+        run: mvn -B -nsu -DbackwardCompatTests -DfailIfNoTests -pl :backward-compat-current-server-old-clients test
+
+      - name: Test progressive upgrade
+        run: mvn -B -nsu -DbackwardCompatTests -DfailIfNoTests -pl :upgrade test
+
+      - name: Other tests
+        run: |
+          mvn -B -nsu -DbackwardCompatTests -DfailIfNoTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct test
+
+      - name: Upload container logs on failure
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
+        continue-on-error: true
+        with:
+          retention-days: 7
+          name: backward-compatibility-tests-container-logs
+          if-no-files-found: ignore
+          path: |
+            **/docker.log
+
+      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
+        if: ${{ always() }}
+        uses: ./.github/actions/copy-test-reports
+
+      - name: Publish Test Report
+        uses: apache/pulsar-test-infra/action-junit-report@master
+        if: ${{ always() }}
+        with:
+          report_paths: 'surefire-reports/TEST-*.xml'
+          annotate_only: 'true'
+
+      - name: Upload Surefire reports
+        uses: actions/upload-artifact@v4
+        if: failure()
+        continue-on-error: true
+        with:
+          name: backward-compatibility-tests-reports
+          path: surefire-reports
+          if-no-files-found: ignore
+          retention-days: 7
+
+  windows-build:
+    name: Build with windows on JDK 11
+    runs-on: windows-latest
+    timeout-minutes: 30
+    needs: [ 'build-and-license-check' ]
+    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Install mingw
+        run: choco install mingw
+
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 11
+
+      - name: mvn package
+        run: mvn -B -nsu clean package -DskipTests
+
+  macos-build:
+    name: Build with macos on JDK 11
+    runs-on: macos-latest
+    timeout-minutes: 30
+    needs: [ 'build-and-license-check' ]
+    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 11
+
+      - name: mvn package
+        run: mvn -B -nsu clean package -DskipTests
+
+  jdk-compatibility-checks:
+    name: ${{ matrix.step_name }}
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ matrix.timeout || 60 }}
+    needs: [ 'build-and-license-check' ]
+    if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - step_name: Compatibility Check Java8
+            jdk_version: 8
+          - step_name: Compatibility Check Java11
+            jdk_version: 11
+          - step_name: Compatibility Check Java17
+            jdk_version: 17
+          - step_name: Compatibility Check Java21
+            jdk_version: 21
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+
+      - name: Set up JDK ${{ matrix.jdk_version }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.jdk_version }}
+
+      - name: Build with Maven
+        run: mvn clean package -B -nsu -DskipBookKeeperServerTests
+
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps
+
+  typo-check:
+    name: Typo Check
+    # only run on pull requests because of security reasons
+    # we shouldn't trust external actions for builds within the repository
+    if: ${{ github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check typos
+        uses: crate-ci/typos@v1.22.4
+
+  owasp-dependency-check:
+    name: OWASP Dependency Check
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: [ 'build-and-license-check' ]
+    if: ${{ needs.build-and-license-check.outputs.need_owasp == 'true' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 21
+
+      - name: run "clean install verify" to trigger dependency check
+        # excluding dlfs because it includes hadoop lib with
+        # CVEs that we cannot patch up anyway
+        run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs,!tests'
+
+      - name: Upload report
+        uses: actions/upload-artifact@v4
+        if: ${{ cancelled() || failure() }}
+        continue-on-error: true
+        with:
+          name: dependency report
+          path: target/dependency-check-report.html
+          retention-days: 7
+
+  bookkeeper-ci-checks-completed:
+    name: "BookKeeper CI checks completed"
+    if: ${{ always() && ((github.event_name != 'schedule') || (github.repository == 'apache/bookkeeper')) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: [
+      'backward-compatibility-tests',
+      'build-and-license-check',
+      'integration-tests',
+      'jdk-compatibility-checks',
+      'macos-build',
+      'owasp-dependency-check',
+      'typo-check',
+      'unit-tests',
+      'windows-build'
+    ]
+    steps:
+      - name: Check build-and-license-check success
+        run: |
+          if [[ ! ( \
+                   "${{ needs.build-and-license-check.result }}" == "success" \
+               ) ]]; then
+            echo "Required jobs haven't been completed successfully."
+            exit 1
+          fi
+      - name: Check typo-check success for pull requests
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          if [[ ! ( \
+                   "${{ needs.typo-check.result }}" == "success" \
+               ) ]]; then
+            echo "Required jobs haven't been completed successfully."
+            exit 1
+          fi
+      - name: Check that other required jobs were completed successfully
+        if: ${{ needs.build-and-license-check.outputs.docs_only != 'true' }}
+        run: |
+          if [[ ! ( \
+                   "${{ needs.backward-compatibility-tests.result }}" == "success" \
+                && "${{ needs.integration-tests.result }}" == "success" \
+                && "${{ needs.jdk-compatibility-checks.result }}" == "success" \
+                && "${{ needs.macos-build.result }}" == "success" \
+                && "${{ needs.unit-tests.result }}" == "success" \
+                && "${{ needs.windows-build.result }}" == "success" \
+               ) ]]; then
+            echo "Required jobs haven't been completed successfully."
+            exit 1
+          fi

.github/workflows/bk-streamstorage-python.yml

@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: BookKeeper StreamStorage Python Client
+on:
+  pull_request:
+    branches:
+      - master
+      - branch-*
+    paths:
+      - 'stream/**'
+      - '.github/workflows/bk-streamstorage-python.yml'
+  push:
+    branches:
+      - master
+      - branch-*
+    paths:
+      - 'stream/**'
+      - '.github/workflows/bk-streamstorage-python.yml'
+
+jobs:
+  stream-storage-python-client-unit-tests:
+    name: StreamStorage Python Client Unit Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+      - name: Test
+        run: ./stream/clients/python/scripts/test.sh
+
+
+  Stream-storage-python-client-integration-tests:
+    name: StreamStorage Python Client Integration Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 11
+      - name: Build
+        run: mvn -q -T 1C -B -nsu clean install -DskipTests -Dcheckstyle.skip -Dspotbugs.skip -Drat.skip -Dmaven.javadoc.skip
+      - name: Pick ubuntu mirror for the docker image build
+        run: |
+          # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
+          $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror
+      - name: Build Test image
+        run: ./stream/clients/python/docker/build-local-image.sh
+      - name: Test
+        run: ./stream/clients/python/scripts/docker_integration_tests.sh
+
+

.github/workflows/bot.yml

@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Bot tests
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  bot:
+    name: Bot tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: clone repository
+        uses: actions/checkout@v4
+
+      - name: bot actions
+        uses: actions/github-script@v7
+        env:
+          RERUN_CMD: 'rerun failure checks'
+        with:
+          github-token: ${{secrets.BKBOT_TOKEN}}
+          script: |
+            const path = require('path')
+            const scriptPath = path.resolve('.github/actions/bot/src/run.js')
+            require(scriptPath)({core}, {context}, {github})

.github/workflows/ci-bkbot.yaml

@@ -0,0 +1,298 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Description:
+# This GitHub Actions workflow enables rerunning CI via PR/Issue comments using the /bkbot command.
+# Supported commands: /bkbot rerun [keyword]
+# - /bkbot rerun           => Rerun the latest run of each workflow under the same head SHA, limited to runs with a conclusion of failure/cancelled/timed_out/skipped (entire run).
+# - /bkbot rerun <keyword> => Regardless of workflow/job status, fetch all jobs in the latest runs, match by name, and rerun each matching job.
+# Logging instructions:
+# - Jobs that are failed/cancelled/timed_out/skipped are scanned from all the latest workflow runs (including those in progress), thus jobs fail/skipped during progress can be captured.
+# Triggering condition: When a new comment is created containing /bkbot.
+
+name: BookKeeper Bot
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  actions: write
+  contents: read
+
+jobs:
+  bkbot:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    if: github.event_name == 'issue_comment' && contains(github.event.comment.body, '/bkbot')
+    steps:
+      - name: Execute bkbot command
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // Supported commands:
+            // - /bkbot rerun
+            //   Reruns all completed workflows with conclusions of failure/timed_out/skipped/cancelled
+            //   If workflow is still running, cannot rerun whole workflow, just suggest using "/bkbot rerun jobname"
+            // - /bkbot rerun jobname
+            //   Matches job.name by keyword, reruns matching jobs (regardless of current state, failures are logged)
+            // - /bkbot stop or /bkbot cancel
+            //   Cancels all still running (queued/in_progress) workflow runs associated with the current PR
+
+            const commentBody = context.payload.comment.body.trim();
+            const prefix = '/bkbot';
+            if (!commentBody.startsWith(prefix)) {
+              console.log('Not a bkbot command, skipping ...');
+              return;
+            }
+
+            if (!context.payload.issue || !context.payload.issue.pull_request) {
+              console.error('This comment is not on a Pull Request. bkbot only works on PRs.');
+              return;
+            }
+
+            const parts = commentBody.split(/\s+/);
+            const sub = (parts[1] || '').toLowerCase();
+            const arg = parts.length > 2 ? parts.slice(2).join(' ') : '';
+
+            const supported = ['rerun', 'stop', 'cancel'];
+            if (!supported.includes(sub)) {
+              console.log(`Unsupported command '${sub}'. Supported: '/bkbot rerun [jobName?]', '/bkbot stop', '/bkbot cancel'.`);
+              return;
+            }
+
+            const prNum = context.payload.issue.number;
+
+            // Get PR info
+            let pr;
+            try {
+              ({ data: pr } = await github.rest.pulls.get({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                pull_number: prNum
+              }));
+            } catch (e) {
+              console.error(`Failed to fetch PR #${prNum}: ${e.message}`);
+              return;
+            }
+
+            const headSha = pr.head.sha;
+            const prBranch = pr.head.ref;
+            const prUser = (pr.head && pr.head.user && pr.head.user.login) ? pr.head.user.login : pr.user.login;
+            const prUrl = pr.html_url;
+
+            console.log(`bkbot handling PR #${prNum} ${prUrl}`);
+            console.log(`PR branch='${prBranch}', headSha='${headSha}', author='${prUser}'`);
+            console.log(`Command parsed => sub='${sub}', arg='${arg || ''}'`);
+
+            // Fetch workflow runs in this repo triggered by this user on this branch, then filter by headSha
+            let page = 1;
+            const allRunsRaw = [];
+            while (true) {
+              const { data } = await github.rest.actions.listWorkflowRunsForRepo({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                actor: prUser,
+                branch: prBranch,
+                per_page: 100,
+                page
+              });
+              const wr = data.workflow_runs || [];
+              if (wr.length === 0) break;
+              allRunsRaw.push(...wr);
+              if (wr.length < 100) break;
+              page++;
+            }
+
+            const runsAtHead = allRunsRaw.filter(r => r.head_sha === headSha);
+            if (runsAtHead.length === 0) {
+              console.error(`No workflow runs found for head SHA ${headSha} on branch ${prBranch}.`);
+              return;
+            }
+
+            // Only keep the latest run for each workflow_id
+            runsAtHead.sort((a, b) => {
+              if (a.workflow_id !== b.workflow_id) return a.workflow_id - b.workflow_id;
+              return new Date(b.created_at) - new Date(a.created_at);
+            });
+            const latestRuns = [];
+            const seen = new Set();
+            for (const r of runsAtHead) {
+              if (!seen.has(r.workflow_id)) {
+                seen.add(r.workflow_id);
+                latestRuns.push(r);
+              }
+            }
+
+            function runKey(r) {
+              return `[run_id=${r.id}] ${r.name || '(unnamed)'} | status=${r.status} | conclusion=${r.conclusion || '-'} | ${r.html_url}`;
+            }
+
+            console.log('--- Latest workflow runs for this PR headSHA (one per workflow) ---');
+            for (const r of latestRuns) console.log('- ' + runKey(r));
+
+            // Utility: list all jobs in a run
+            async function listAllJobs(runId) {
+              let jobs = [];
+              let p = 1;
+              while (true) {
+                const { data } = await github.rest.actions.listJobsForWorkflowRun({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  run_id: runId,
+                  per_page: 100,
+                  page: p
+                });
+                const js = data.jobs || [];
+                if (js.length === 0) break;
+                jobs.push(...js);
+                if (js.length < 100) break;
+                p++;
+              }
+              return jobs;
+            }
+
+            // Utility: rerun a single job
+            async function rerunJob(job, run) {
+              try {
+                if (github.rest.actions.reRunJobForWorkflowRun) {
+                  await github.rest.actions.reRunJobForWorkflowRun({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    job_id: job.id
+                  });
+                } else {
+                  await github.request('POST /repos/{owner}/{repo}/actions/jobs/{job_id}/rerun', {
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    job_id: job.id
+                  });
+                }
+                console.log(`Re-ran job '${job.name}' (job_id=${job.id}) in run '${run.name}' | ${run.html_url}`);
+                return true;
+              } catch (e) {
+                console.log(`Failed to re-run job '${job.name}' (job_id=${job.id}) in run '${run.name}': ${e.message}`);
+                return false;
+              }
+            }
+
+            // Command 1: /bkbot rerun
+            if (sub === 'rerun' && !arg) {
+              const targetConclusions = new Set(['failure', 'timed_out', 'cancelled', 'skipped']);
+              let fullRerunCount = 0;
+              let skippedRunning = 0;
+              let skippedConclusion = 0;
+
+              console.log('Mode: full workflow re-run for completed runs with conclusions in [failure,timed_out,cancelled,skipped].');
+              for (const r of latestRuns) {
+                if (r.status !== 'completed') {
+                  console.log(`Skip (still running) ${runKey(r)}. Cannot re-run whole workflow. Consider '/bkbot rerun <jobName>' for single job.`);
+                  skippedRunning++;
+                  continue;
+                }
+                if (!targetConclusions.has(r.conclusion)) {
+                  console.log(`Skip (conclusion not eligible) ${runKey(r)}`);
+                  skippedConclusion++;
+                  continue;
+                }
+                try {
+                  await github.rest.actions.reRunWorkflow({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    run_id: r.id
+                  });
+                  console.log(`Triggered full re-run for ${runKey(r)}`);
+                  fullRerunCount++;
+                } catch (e) {
+                  console.log(`Failed to trigger full re-run for ${runKey(r)}: ${e.message}`);
+                }
+              }
+
+              if (fullRerunCount === 0) {
+                console.error(`No eligible workflow runs to re-run. Skipped running=${skippedRunning}, skipped by conclusion=${skippedConclusion}.`);
+              } else {
+                console.log(`Finished. Triggered full re-run for ${fullRerunCount} workflow run(s). Skipped running=${skippedRunning}, skipped by conclusion=${skippedConclusion}.`);
+              }
+              return;
+            }
+
+            // Command 2: /bkbot rerun jobname
+            if (sub === 'rerun' && arg) {
+              const keyword = arg.trim();
+              console.log(`Mode: job-level re-run. keyword='${keyword}'`);
+
+              let matchedJobs = 0;
+              let successJobs = 0;
+
+              for (const r of latestRuns) {
+                let jobs = [];
+                try {
+                  jobs = await listAllJobs(r.id);
+                } catch (e) {
+                  console.log(`Failed to list jobs for ${runKey(r)}: ${e.message}`);
+                  continue;
+                }
+                for (const j of jobs) {
+                  if (j.name && j.name.includes(keyword)) {
+                    matchedJobs++;
+                    const ok = await rerunJob(j, r);
+                    if (ok) successJobs++;
+                  }
+                }
+              }
+
+              if (matchedJobs === 0) {
+                console.error(`No jobs matched keyword '${keyword}' among latest runs for this PR head.`);
+              } else {
+                console.log(`Finished. Matched ${matchedJobs} job(s); successfully requested re-run for ${successJobs} job(s).`);
+              }
+              return;
+            }
+
+            // Command 3: /bkbot stop or /bkbot cancel
+            if (sub === 'stop' || sub === 'cancel') {
+              console.log('Mode: cancel running workflow runs (queued/in_progress).');
+
+              let cancelCount = 0;
+              let alreadyCompleted = 0;
+
+              for (const r of latestRuns) {
+                if (r.status === 'completed') {
+                  console.log(`Skip (already completed) ${runKey(r)}`);
+                  alreadyCompleted++;
+                  continue;
+                }
+                try {
+                  await github.rest.actions.cancelWorkflowRun({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    run_id: r.id
+                  });
+                  console.log(`Cancel requested for ${runKey(r)}`);
+                  cancelCount++;
+                } catch (e) {
+                  console.log(`Failed to cancel ${runKey(r)}: ${e.message}`);
+                }
+              }
+
+              if (cancelCount === 0) {
+                console.error(`No running workflow runs to cancel. Already completed: ${alreadyCompleted}.`);
+              } else {
+                console.log(`Finished. Requested cancel for ${cancelCount} running workflow run(s). Already completed: ${alreadyCompleted}.`);
+              }
+              return;
+            }

.github/workflows/codeql.yml

@@ -0,0 +1,97 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  schedule:
+    - cron: '27 11 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 360
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+      pull-requests: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'c-cpp', 'java-kotlin', 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Detect changed files
+      id: changes
+      uses: apache/pulsar-test-infra/paths-filter@master
+      with:
+        filters: .github/changes-filter.yaml
+        list-files: csv
+
+    - name: Check changed files
+      id: check_changes
+      run: |
+        echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
+
+    - name: Cache local Maven repository
+      if: steps.check_changes.outputs.docs_only != 'true'
+      id: cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.m2/repository/*/*/*
+          !~/.m2/repository/org/apache/bookkeeper
+          !~/.m2/repository/org/apache/distributedlog
+        key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Set up JDK 11
+      if: steps.check_changes.outputs.docs_only != 'true'
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: 11
+
+    - name: Validate pull request
+      if: steps.check_changes.outputs.docs_only != 'true'
+      run: |
+        mvn -T 1C -B -nsu clean install -Ddistributedlog -DskipTests
+
+    - name: Perform CodeQL Analysis
+      if: steps.check_changes.outputs.docs_only != 'true'
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/dead-link-checker.yaml

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Dead link checker
+
+on:
+  push:
+  pull_request:
+    branches:
+      - master
+      - branch-*
+    paths:
+      - '**.md'
+
+concurrency:
+  group: dlc-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-dead-links:
+    name: Dead link checker
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo npm install -g markdown-link-check@3.11.2
+      - run: |
+          for file in $(find . -name "*.md"); do
+            markdown-link-check -c .dlc.json -q "$file"
+          done

.github/workflows/java21-daily-build.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: JDK 21 Daily Build
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs at 00:00 UTC every day
+  workflow_dispatch:
+
+jobs:
+  jdk21-daily-build:
+    name: Build on JDK 21
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 21
+      - name: Build with Maven
+        run: mvn -B clean install
+      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure
+        if: failure()
+        continue-on-error: true
+        uses: ./.github/actions/copy-test-reports
+      - name: Upload Surefire reports
+        uses: actions/upload-artifact@v4
+        if: failure()
+        continue-on-error: true
+        with:
+          name: jdk21-tests-reports
+          path: surefire-reports

.github/workflows/owasp-daily-build.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: OWASP Daily Build
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs at 00:00 UTC every day
+  workflow_dispatch:
+
+env:
+  NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
+
+jobs:
+  owasp-daily-build:
+    name: OWASP Dependency Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 21
+
+      - name: run "clean install verify" to trigger dependency check
+        # excluding dlfs because it includes hadoop lib with
+        # CVEs that we cannot patch up anyway
+        run: mvn -q -B -ntp clean install verify -Powasp-dependency-check -DskipTests -pl '!stream/distributedlog/io/dlfs,!tests'

.github/workflows/website-deploy.yaml

@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Website deploy
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+    paths:
+      - 'site3/**'
+      - '.github/workflows/website-deploy.yaml'
+
+env:
+  DEPLOY_URL: "https://bookkeeper.apache.org/"
+
+jobs:
+  build-website:
+    name: Build and deploy the website
+    if: ${{ github.repository == 'apache/bookkeeper' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 11
+
+
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: '16'
+
+      - name: Setup yarn
+        run: npm install -g yarn
+
+      - name: Publish
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: |
+          ./site3/website/scripts/build-website.sh
+          ./site3/website/scripts/publish-website.sh

.github/workflows/website-pr-validation.yml

@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Website PR validation
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - master
+      - branch-*
+    paths:
+      - 'site3/**'
+      - '.github/workflows/website-pr-validation.yaml'
+
+jobs:
+  website-pull-validation:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 11
+
+
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: '16'
+
+      - name: Setup yarn
+        run: npm install -g yarn
+
+      - name: Build website
+        run: |
+          ./site3/website/scripts/build-website.sh

.github/workflows/windows-daily-build.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Windows Daily Build
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs at 00:00 UTC every day
+  workflow_dispatch:
+
+jobs:
+  windows-daily-build:
+    name: Daily Build and Test on Windows
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 21
+      - name: Build with Maven
+        run: mvn -B clean install
+      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure
+        if: failure()
+        continue-on-error: true
+        uses: ./.github/actions/copy-test-reports
+      - name: Upload Surefire reports
+        uses: actions/upload-artifact@v4
+        if: failure()
+        continue-on-error: true
+        with:
+          name: windows-tests-reports
+          path: surefire-reports

.gitignore

@@ -0,0 +1,50 @@
+# Eclipse
+.classpath
+.project
+.settings/
+.recommenders/
+
+# IntelliJ
+.idea/
+!.idea/icon.svg
+!.idea/vcs.xml
+*.iml
+*.iws
+
+# NetBeans
+/**/nb-configuration.xml
+
+# Mac
+.DS_Store
+
+# Maven
+lib/
+log/
+target/
+dependency-reduced-pom.xml
+
+# Logs
+logs/
+
+# Vagrant
+**/.vagrant
+
+# Data directory
+data/
+
+# Pid files
+**/*.pid
+
+# files are generated under following directories
+tools/all/src/main/resources
+
+# Exclude versionBackup file (generated by `mvn versions:set`)
+**/*.versionsBackup
+
+node_modules
+package-lock.json
+# gradle
+build/
+.gradle/
+*.log
+*.dat

.test-infra/scripts/post-docker-tests.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -ex
+kill $(cat docker-log.pid) || true

.test-infra/scripts/pre-docker-tests.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -ex
+
+id
+ulimit -a
+pwd
+df -Th
+ps -eo euser,pid,ppid,pgid,start,pcpu,pmem,cmd
+docker info
+docker system prune -f
+# clean up any dangling networks from previous runs
+docker network prune -f --filter "until=12h"
+docker system events > docker.debug-info & echo $! > docker-log.pid
+docker pull quay.io/coreos/etcd:v3.5.14

.test-infra/scripts/slack-email-digest/.gitignore

@@ -0,0 +1 @@
+configuration.yaml

.test-infra/scripts/slack-email-digest/README.md

@@ -0,0 +1,4 @@
+# Slack to email digest bot
+
+This script is responsible for generating daily per channel digests and emailing them to certail mailing lists.
+

.test-infra/scripts/slack-email-digest/configuration-example.yaml

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+slack:
+    token: xoxp-xxxxxxxxxxxxxx
+
+mail:
+    fromAddress: '"My Slack" <my.slack@gmail.com>'
+    smtp: smtp.gmail.com:587
+    useTLS: true
+    username: my.slack@gmail.com
+    password: xxxxxxxxx
+
+channels:
+    # Maps channels to a particular email address
+    general: users@my.project.org
+    dev: dev@my.project.org
+    random: users@my.project.org

.test-infra/scripts/slack-email-digest/requirements.txt

@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+slacker
+PyYAML

.test-infra/scripts/slack-email-digest/slack_email_digest.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import slacker
+import yaml
+import time
+import re
+import datetime
+import smtplib
+from email.mime.text import MIMEText
+
+
+conf = yaml.load(open('configuration.yaml'))
+
+
+def send_digest(channel, address, digest):
+    msg = MIMEText(digest, _charset='utf-8')
+    msg['From'] = conf['mail']['fromAddress']
+    msg['To'] = address
+    msg['Subject'] = 'Slack digest for #%s - %s' % (
+        channel, datetime.datetime.now().strftime('%Y-%m-%d'))
+    server = smtplib.SMTP(conf['mail']['smtp'])
+    if conf['mail']['useTLS']:
+        server.starttls()
+    if 'username' in conf['mail']:
+        server.login(conf['mail']['username'], conf['mail']['password'])
+
+    server.sendmail(conf['mail']['fromAddress'], address, msg.as_string())
+    server.quit()
+
+
+slack = slacker.Slacker(conf['slack']['token'])
+
+channels = slack.channels.list().body['channels']
+
+# Get a mapping between Slack internal user ids and real names
+users = {}
+for user in slack.users.list().body['members']:
+    real_name = user.get('real_name', user.get('name'))
+    users[user['id']] = real_name
+
+last_day_timestamp = time.time() - (24 * 3600)
+
+for channel in channels:
+    id = channel['id']
+    name = channel['name']
+    topic = channel['topic']['value']
+
+    if name not in conf['channels']:
+        print('Ignoring channel: #%s' % name)
+        continue
+
+    toAddress = conf['channels'][name]
+    print('Getting digest of #%s --> %s' % (name, toAddress))
+
+    messages = slack.channels.history(channel=id,
+                                      oldest=last_day_timestamp,
+                                      count=1000)
+    digest = ''
+    for m in reversed(messages.body['messages']):
+        if not m['type'] == 'message':
+            continue
+
+        user = m.get('user')
+        if not user:
+            user = m['comment']['user']
+        sender = users.get(user, '')
+
+        date = datetime.datetime.utcfromtimestamp(float(m['ts'])).strftime('%Y-%m-%d %H:%M:%S UTC')
+        # Replace users id mentions with real names
+        text = re.sub(r'<@(\w+)>', lambda m: '@' + users[m.group(1)], m['text'])
+
+        digest += '%s - %s: %s\n' % (date, sender, text)
+        for reaction in m.get('reactions', []):
+            digest += '%s : %s\n' % (reaction['name'], ', '.join(map(users.get, reaction['users'])))
+        digest += '----\n'
+
+    if digest:
+        send_digest(name, toAddress, digest)

.typos.toml

@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+[default.extend-words]
+# abbr
+"ba" = "ba"
+"bve" = "bve"
+"cace" = "cace"
+"cann" = "cann"
+"dbe" = "dbe"
+"entrys" = "entrys"
+"fo" = "fo"
+"ine" = "ine"
+"isse" = "isse"
+"mor" = "mor"
+"nwe" = "nwe"
+"nd" = "nd"
+"nin" = "nin"
+"oce" = "oce"
+"ot" = "ot"
+"ser" = "ser"
+"shouldnot" = "shouldnot"
+"tio" = "tio"
+"ue" = "ue"
+# keep for comptability
+"deleteable" = "deleteable"
+"infinit" = "infinit"
+"explict" = "explict"
+"uninitalize" = "uninitalize"
+# keyword fp
+"guage" = "guage"
+"passin" = "passin"
+"testng" = "testng"
+"vertx" = "vertx"
+"verticle" = "verticle"
+
+[files]
+extend-exclude = [
+    "bookkeeper-server/src/test/java/org/apache/bookkeeper/meta/TestLedgerMetadataSerDe.java",
+]

CONTRIBUTING.md

@@ -0,0 +1,25 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+-->
+
+## Contributing to Apache BookKeeper
+The Apache BookKeeper community welcomes contributions from anyone with a passion for distributed systems! BookKeeper has many different opportunities for contributions -- write new examples/tutorials, add new user-facing libraries, work on the core storage components, integrate with different metadata stores (ZooKeeper, Etcd etc), or participate on the documentation effort.
+
+We use a review-then-commit workflow in BookKeeper for all contributions.
+
+We would love for you to contribute to Apache BookKeeper and make it even better!
+Please check the [Contributing to Apache BookKeeper](https://bookkeeper.apache.org/community/contributing/)
+page before starting to work on the project.

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,5 @@
+Apache BookKeeper
+Copyright 2011-2025 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).

README.md

@@ -0,0 +1,53 @@
+<img src="https://pbs.twimg.com/profile_images/545716709311520769/piLLa1iC_400x400.png" alt="logo" style="width: 32px;"/>
+
+[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.bookkeeper/bookkeeper/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.apache.bookkeeper/bookkeeper)
+
+# Apache BookKeeper
+
+Apache BookKeeper is a scalable, fault-tolerant and low latency storage service optimized for append-only workloads.
+
+It is suitable for being used in following scenarios:
+
+- WAL (Write-Ahead-Logging), e.g. HDFS NameNode, Pravega.
+- Message Store, e.g. Apache Pulsar.
+- Offset/Cursor Store, e.g. Apache Pulsar.
+- Object/Blob Store, e.g. storing state machine snapshots.
+
+## Get Started
+
+* Checkout the project [website](https://bookkeeper.apache.org/).
+* *Concepts*: Start with the [basic concepts](https://bookkeeper.apache.org/docs/getting-started/concepts) of Apache BookKeeper.
+  This will help you to fully understand the other parts of the documentation.
+* Follow the [Installation](https://bookkeeper.apache.org/docs/getting-started/installation) guide to set up BookKeeper.
+
+## Documentation
+
+Please visit the [Documentation](https://bookkeeper.apache.org/docs/overview/) from the project website for more information.
+
+## Get In Touch
+
+### Report a Bug
+
+For filing bugs, suggesting improvements, or requesting new features, help us out by [opening a GitHub issue](https://github.com/apache/bookkeeper/issues).
+
+### Need Help?
+
+[Subscribe](mailto:user-subscribe@bookkeeper.apache.org) or [mail](mailto:user@bookkeeper.apache.org) the [user@bookkeeper.apache.org](mailto:user@bookkeeper.apache.org) list - Ask questions, find answers, and also help other users.
+
+[Subscribe](mailto:dev-subscribe@bookkeeper.apache.org) or [mail](mailto:dev@bookkeeper.apache.org) the [dev@bookkeeper.apache.org](mailto:dev@bookkeeper.apache.org) list - Join development discussions, propose new ideas and connect with contributors.
+
+[Join us on Slack](https://communityinviter.com/apps/apachebookkeeper/apache-bookkeeper) - This is the most immediate way to connect with Apache BookKeeper committers and contributors.
+
+## Contributing
+
+We feel that a welcoming open community is important and welcome contributions.
+
+### Contributing Code
+
+1. See our [installation guide](https://bookkeeper.apache.org/docs/next/getting-started/installation/) to get your local environment setup.
+
+2. Take a look at our open issues: [GitHub Issues](https://github.com/apache/bookkeeper/issues).
+
+3. Review our [coding style](https://bookkeeper.apache.org/community/coding-guide/) and follow our [pull requests](https://github.com/apache/bookkeeper/pulls) to learn more about our conventions.
+
+4. Make your changes according to our [contributing guide](https://bookkeeper.apache.org/community/contributing/)

bin/bkctl

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# BookKeeper CLI (experimental)
+
+BINDIR=`dirname "$0"`
+BK_HOME=`cd ${BINDIR}/..;pwd`
+mkdir -p $BK_HOME/logs
+
+source ${BK_HOME}/bin/common.sh
+source ${BK_HOME}/conf/bk_cli_env.sh
+
+CLI_MODULE_PATH=tools/all
+CLI_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-tools"
+CLI_MODULE_HOME=${BK_HOME}/${CLI_MODULE_PATH}
+
+# find the module jar
+CLI_JAR=$(find_module_jar ${CLI_MODULE_PATH} ${CLI_MODULE_NAME})
+
+# set up the classpath
+CLI_CLASSPATH=$(set_module_classpath ${CLI_MODULE_PATH})
+
+DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf
+if [ -z "${CLI_CONF}" ]; then
+  CLI_CONF=${DEFAULT_CONF}
+fi
+
+DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j2.cli.xml
+if [ -z "${CLI_LOG_CONF}" ]; then
+  CLI_LOG_CONF=${DEFAULT_LOG_CONF}
+fi
+CLI_LOG_DIR=${CLI_LOG_DIR:-"$BK_HOME/logs"}
+CLI_LOG_FILE=${CLI_LOG_FILE:-"bkctl.log"}
+CLI_ROOT_LOG_LEVEL=${CLI_ROOT_LOG_LEVEL:-"INFO"}
+CLI_ROOT_LOG_APPENDER=${CLI_ROOT_LOG_APPENDER:-"CONSOLE"}
+
+# Configure the classpath
+CLI_CLASSPATH="$CLI_JAR:$CLI_CLASSPATH:$CLI_EXTRA_CLASSPATH"
+CLI_CLASSPATH="`dirname $CLI_LOG_CONF`:$CLI_CLASSPATH"
+
+# Build the OPTs
+BOOKIE_OPTS=$(build_bookie_opts)
+GC_OPTS=$(build_cli_jvm_opts ${CLI_LOG_DIR} "bkctl-gc.log")
+NETTY_OPTS=$(build_netty_opts)
+LOGGING_OPTS=$(build_cli_logging_opts ${CLI_LOG_CONF} ${CLI_ROOT_LOG_LEVEL} ${CLI_ROOT_LOG_APPENDER} ${CLI_LOG_DIR} ${CLI_LOG_FILE})
+
+OPTS="${OPTS} -cp ${CLI_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS} ${LOGGING_OPTS} ${CLI_EXTRA_OPTS}"
+
+#Change to BK_HOME to support relative paths
+cd "$BK_HOME"
+exec "${JAVA}" ${OPTS} org.apache.bookkeeper.tools.cli.BKCtl --conf ${CLI_CONF} $@

bin/bkperf

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# BookKeeper Perf Tool (experimental)
+
+BINDIR=`dirname "$0"`
+BK_HOME=`cd ${BINDIR}/..;pwd`
+
+source ${BK_HOME}/bin/common.sh
+source ${BK_HOME}/conf/bk_cli_env.sh
+
+CLI_MODULE_PATH=tools/perf
+CLI_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-perf"
+CLI_MODULE_HOME=${BK_HOME}/${CLI_MODULE_PATH}
+
+# find the module jar
+CLI_JAR=$(find_module_jar ${CLI_MODULE_PATH} ${CLI_MODULE_NAME})
+
+# set up the classpath
+CLI_CLASSPATH=$(set_module_classpath ${CLI_MODULE_PATH})
+
+DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf
+if [ -z "${CLI_CONF}" ]; then
+  CLI_CONF=${DEFAULT_CONF}
+fi
+
+DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j2.cli.xml
+if [ -z "${CLI_LOG_CONF}" ]; then
+  CLI_LOG_CONF=${DEFAULT_LOG_CONF}
+fi
+CLI_LOG_DIR=${CLI_LOG_DIR:-"$BK_HOME/logs"}
+CLI_LOG_FILE=${CLI_LOG_FILE:-"bkperf.log"}
+CLI_ROOT_LOG_LEVEL=${CLI_ROOT_LOG_LEVEL:-"INFO"}
+CLI_ROOT_LOG_APPENDER=${CLI_ROOT_LOG_APPENDER:-"CONSOLE"}
+
+mkdir -p ${CLI_LOG_DIR}
+
+# Configure the classpath
+CLI_CLASSPATH="$CLI_JAR:$CLI_CLASSPATH:$CLI_EXTRA_CLASSPATH"
+CLI_CLASSPATH="`dirname $CLI_LOG_CONF`:$CLI_CLASSPATH"
+
+# Build the OPTs
+BOOKIE_OPTS=$(build_bookie_opts)
+GC_OPTS=$(build_cli_jvm_opts ${CLI_LOG_DIR} "bkperf-gc.log")
+NETTY_OPTS=$(build_netty_opts)
+LOGGING_OPTS=$(build_cli_logging_opts ${CLI_LOG_CONF} ${CLI_ROOT_LOG_LEVEL} ${CLI_ROOT_LOG_APPENDER} ${CLI_LOG_DIR} ${CLI_LOG_FILE})
+
+OPTS="${OPTS} -cp ${CLI_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS} ${LOGGING_OPTS} ${CLI_EXTRA_OPTS}"
+
+#Change to BK_HOME to support relative paths
+cd "$BK_HOME"
+exec ${JAVA} ${OPTS} org.apache.bookkeeper.tools.perf.BKPerf --conf ${CLI_CONF} $@

bin/bookkeeper

@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+set -e
+
+BINDIR=`dirname "$0"`
+BK_HOME=`cd ${BINDIR}/..;pwd`
+
+source ${BK_HOME}/bin/common.sh
+
+# default variables
+DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf
+DEFAULT_ZK_CONF=${BK_HOME}/conf/zookeeper.conf
+
+if [ -z "$BOOKIE_CONF" ]; then
+  BOOKIE_CONF_TO_CHECK=${DEFAULT_CONF}
+else
+  BOOKIE_CONF_TO_CHECK=${BOOKIE_CONF}
+fi
+
+FIND_TABLE_SERVICE_RESULT=$(find_table_service ${BOOKIE_CONF_TO_CHECK} $1)
+
+if [ "x${FIND_TABLE_SERVICE_RESULT}" == "xtrue" ]; then
+  BOOKIE_MODULE_PATH=stream/server
+  BOOKIE_MODULE_NAME=${TABLE_SERVICE_MODULE_NAME}
+elif [ "x${FIND_TABLE_SERVICE_RESULT}" == "xfalse" ]; then
+  BOOKIE_MODULE_PATH=bookkeeper-server
+  BOOKIE_MODULE_NAME=${BOOKIE_SERVER_MODULE_NAME}
+else
+  echo ${FIND_TABLE_SERVICE_RESULT}
+  exit 1
+fi
+
+# find the module jar
+BOOKIE_JAR=$(find_module_jar ${BOOKIE_MODULE_PATH} ${BOOKIE_MODULE_NAME})
+
+# set up the classpath
+BOOKIE_CLASSPATH=$(set_module_classpath ${BOOKIE_MODULE_PATH})
+
+bookkeeper_help() {
+    cat <<EOF
+Usage: bookkeeper <command>
+where command is one of:
+
+[service commands]
+
+    bookie              Run a bookie server
+    autorecovery        Run AutoRecovery service
+    zookeeper           Run zookeeper server
+
+[development commands]
+
+    localbookie <n>     Run a test ensemble of <n> bookies locally
+    standalone          Run a standalone cluster (with all service components) locally
+
+[tooling commands]
+
+    upgrade             Upgrade bookie filesystem
+    shell               Run shell for admin commands
+
+[other commands]
+
+    help                This help message
+
+or command is the full name of a class with a defined main() method.
+
+Environment variables:
+   BOOKIE_LOG_CONF        Log4j configuration file (default ${DEFAULT_LOG_CONF})
+   BOOKIE_CONF            Configuration file (default: ${DEFAULT_CONF})
+   BOOKIE_ZK_CONF         Configuration file for zookeeper (default: $DEFAULT_ZK_CONF)
+   BOOKIE_EXTRA_OPTS      Extra options to be passed to the jvm
+   BOOKIE_EXTRA_CLASSPATH Add extra paths to the bookkeeper classpath
+   BOOKIE_PID_DIR         Folder where the Bookie server PID file should be stored
+   BOOKIE_STOP_TIMEOUT    Wait time before forcefully kill the Bookie server instance, if the stop is not successful
+
+These variable can also be set in conf/bkenv.sh
+EOF
+}
+
+# if no args specified, show usage
+if [ $# = 0 ]; then
+  bookkeeper_help;
+  exit 1;
+fi
+
+# get arguments
+COMMAND=$1
+shift
+
+LOCALBOOKIES_CONFIG_DIR="${LOCALBOOKIES_CONFIG_DIR:-/tmp/localbookies-config}"
+if [ ${COMMAND} == "shell" ]; then
+  DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j2.shell.xml
+  if [[ $1 == "-localbookie"  ]]; then
+    if [[ $2 == *:* ]];
+    then
+      BOOKIE_CONF=${LOCALBOOKIES_CONFIG_DIR}/$2.conf
+      shift 2
+    else
+      BOOKIE_CONF=${LOCALBOOKIES_CONFIG_DIR}/baseconf.conf
+      shift
+    fi
+  fi
+fi
+
+if [ -z "$BOOKIE_ZK_CONF" ]; then
+    BOOKIE_ZK_CONF=$DEFAULT_ZK_CONF
+fi
+
+if [ -z "$BOOKIE_CONF" ]; then
+  BOOKIE_CONF=${DEFAULT_CONF}
+fi
+
+# Configure logging
+if [ -z "$BOOKIE_LOG_CONF" ]; then
+  BOOKIE_LOG_CONF=${DEFAULT_LOG_CONF}
+fi
+BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"$BK_HOME/logs"}
+BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"bookkeeper-server.log"}
+BOOKIE_ROOT_LOG_LEVEL=${BOOKIE_ROOT_LOG_LEVEL:-"INFO"}
+BOOKIE_ROOT_LOG_APPENDER=${BOOKIE_ROOT_LOG_APPENDER:-"CONSOLE"}
+
+# Configure the classpath
+BOOKIE_CLASSPATH="$BOOKIE_JAR:$BOOKIE_CLASSPATH:$BOOKIE_EXTRA_CLASSPATH"
+BOOKIE_CLASSPATH="`dirname $BOOKIE_LOG_CONF`:$BOOKIE_CLASSPATH"
+
+# Build the OPTS
+BOOKIE_OPTS=$(build_bookie_opts)
+GC_OPTS=$(build_bookie_jvm_opts ${BOOKIE_LOG_DIR} "gc_%p.log")
+NETTY_OPTS=$(build_netty_opts)
+LOGGING_OPTS=$(build_logging_opts ${BOOKIE_LOG_CONF} ${BOOKIE_ROOT_LOG_LEVEL} ${BOOKIE_ROOT_LOG_APPENDER} ${BOOKIE_LOG_DIR} ${BOOKIE_LOG_FILE})
+
+BOOKIE_EXTRA_OPTS="${BOOKIE_EXTRA_OPTS} -Dorg.bouncycastle.fips.approved_only=true"
+OPTS="${OPTS} -cp ${BOOKIE_CLASSPATH} ${BOOKIE_OPTS} ${GC_OPTS} ${NETTY_OPTS} ${LOGGING_OPTS} ${BOOKIE_EXTRA_OPTS}"
+
+# Create log dir if it doesn't exist
+if [ ! -d ${BOOKIE_LOG_DIR} ]; then
+    mkdir ${BOOKIE_LOG_DIR}
+fi
+
+#Change to BK_HOME to support relative paths
+cd "$BK_HOME"
+if [ ${COMMAND} == "bookie" ]; then
+  exec "${JAVA}" ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.server.Main --conf ${BOOKIE_CONF} $@
+elif [ ${COMMAND} == "autorecovery" ]; then
+  exec "${JAVA}" ${OPTS} ${JMX_ARGS} org.apache.bookkeeper.replication.AutoRecoveryMain --conf ${BOOKIE_CONF} $@
+elif [ ${COMMAND} == "localbookie" ]; then
+  NUMBER=$1
+  shift
+  exec "${JAVA}" ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.util.LocalBookKeeper ${NUMBER} ${BOOKIE_CONF} $@
+elif [ ${COMMAND} == "standalone" ]; then
+  exec "${JAVA}" ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.bookkeeper.stream.cluster.StandaloneStarter --conf ${BK_HOME}/conf/standalone.conf $@
+elif [ ${COMMAND} == "upgrade" ]; then
+  exec "${JAVA}" ${OPTS} org.apache.bookkeeper.bookie.FileSystemUpgrade --conf ${BOOKIE_CONF} $@
+elif [ $COMMAND == "zookeeper" ]; then
+    BOOKIE_LOG_FILE=${BOOKIE_LOG_FILE:-"zookeeper.log"}
+    exec "${JAVA}" $OPTS -Dbookkeeper.log.file=$BOOKIE_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $BOOKIE_ZK_CONF $@
+elif [ ${COMMAND} == "shell" ]; then
+  exec "${JAVA}" ${OPTS} org.apache.bookkeeper.bookie.BookieShell -conf ${BOOKIE_CONF} $@
+elif [ ${COMMAND} == "help" ]; then
+  bookkeeper_help;
+else
+  exec "${JAVA}" ${OPTS} ${COMMAND} $@
+fi
+

bin/bookkeeper-cluster.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+
+BINDIR=`dirname "$0"`
+BK_HOME=`cd $BINDIR/..;pwd`
+if [ -f $BK_HOME/conf/bkenv.sh ]
+then
+ . $BK_HOME/conf/bkenv.sh
+fi
+
+BKCFG=$BK_HOME/conf/bk_server.conf
+CLUSTER=$BK_HOME/conf/bookies
+usage() {
+    cat <<EOF
+Usage: bookkeeper-cluster.sh (start|stop|kill)
+
+The list of hosts in the cluster must be available in
+$CLUSTER
+with one hostname per line.
+
+BookKeeper must be installed in the same location on each host.
+EOF
+}
+
+if [ ! -f $CLUSTER ]; then
+  echo -e "\nCluster file ($CLUSTER) does not exist\n"
+  usage
+  exit 1
+fi
+
+NUMHOSTS=$(wc -l $CLUSTER | awk '{print $1}')
+if [ "$NUMHOSTS" = "0" ]; then
+  echo -e "\nCluster file ($CLUSTER) is empty\n"
+  usage
+  exit 1
+fi
+
+bookies_list() {
+  $BINDIR/bookkeeper shell listbookies 2> /dev/null
+}
+
+bookies_available() {
+  bookies_list | wc -l
+}
+
+start() {
+  for B in `cat $CLUSTER`; do
+    echo "Starting bookie on $B"
+    ssh $B $BINDIR/bookkeeper-daemon.sh start bookie
+  done
+
+  BOOKIESSTARTED=0
+  COUNT=0
+
+  while [ $BOOKIESSTARTED -lt $NUMHOSTS ];  do
+    sleep 1
+    COUNT=$(($COUNT+1))
+    if [ $COUNT = 20 ]; then
+      echo "Could not start all bookies"
+      exit 1
+    fi
+
+    BOOKIESSTARTED=$(bookies_available)
+
+    echo "$BOOKIESSTARTED bookies started"
+  done
+}
+
+stop() {
+  for B in `cat $CLUSTER`; do
+    echo "Stopping bookie on $B"
+    ssh $B $BINDIR/bookkeeper-daemon.sh stop bookie $FORCE
+  done
+
+  COUNT=0
+  BOOKIESSTARTED=$NUMHOSTS
+  while [ $BOOKIESSTARTED -gt 0 ];  do
+    sleep 1
+
+    COUNT=$((COUNT+1))
+    if [ $COUNT = 20 ]; then
+      echo "Couldn not stop all bookies. $BOOKIESSTARTED still running"
+      exit 2
+    fi
+
+    BOOKIESSTARTED=$(bookies_available)
+  done
+}
+
+status() {
+  BOOKIESSTARTED=$(bookies_available)
+  echo "$BOOKIESSTARTED bookies running"
+  COUNT=1
+  for b in $(bookies_list); do
+    echo "$COUNT: $b"
+    COUNT=$(($COUNT+1))
+  done
+}
+
+case $1 in
+    start)
+      start
+      ;;
+    stop)
+      stop
+      ;;
+    kill)
+      FORCE="-force"
+      stop
+      ;;
+    status)
+      status
+      ;;
+    *)
+      usage
+      ;;
+esac

bin/bookkeeper-daemon.sh

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+usage() {
+    cat <<EOF
+Usage: bookkeeper-daemon.sh (start|stop) <command> <args...>
+where command is one of:
+    bookie           Run the bookie server
+    autorecovery     Run the AutoRecovery service daemon
+    zookeeper        Run the zookeeper server
+
+where argument is one of:
+    -force (accepted only with stop command): Decides whether to stop the Bookie Server forcefully if not stopped by normal shutdown
+EOF
+}
+
+BINDIR=$(dirname "$0")
+BK_HOME=$(cd $BINDIR/..;pwd)
+
+if [ -f $BK_HOME/conf/bkenv.sh ]
+then
+ . $BK_HOME/conf/bkenv.sh
+fi
+
+BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"$BK_HOME/logs"}
+
+BOOKIE_ROOT_LOG_LEVEL=${BOOKIE_ROOT_LOG_LEVEL:-'INFO'}
+BOOKIE_ROOT_LOG_APPENDER=${BOOKIE_ROOT_LOG_APPENDER:-'ROLLINGFILE'}
+
+BOOKIE_STOP_TIMEOUT=${BOOKIE_STOP_TIMEOUT:-30}
+
+BOOKIE_PID_DIR=${BOOKIE_PID_DIR:-$BK_HOME/bin}
+
+if [ $# -lt 2 ]
+then
+  echo "Error: no enough arguments provided."
+  usage
+  exit 1
+fi
+
+startStop=$1
+shift
+command=$1
+shift
+
+case $command in
+  (zookeeper)
+    echo "doing $startStop $command ..."
+    ;;
+  (bookie)
+    echo "doing $startStop $command ..."
+    ;;
+  (autorecovery)
+    echo "doing $startStop $command ..."
+    ;;
+  (standalone)
+    echo "doing $startStop $command ..."
+    ;;
+  (*)
+    echo "Error: unknown service name $command"
+    usage
+    exit 1
+    ;;
+esac
+
+export BOOKIE_LOG_DIR=$BOOKIE_LOG_DIR
+export BOOKIE_ROOT_LOG_LEVEL=$BOOKIE_ROOT_LOG_LEVEL
+export BOOKIE_ROOT_LOG_APPENDER=$BOOKIE_ROOT_LOG_APPENDER
+export BOOKIE_LOG_FILE=bookkeeper-$command-$HOSTNAME.log
+
+pid_file="${BOOKIE_PID_DIR}/bookkeeper-${command}.pid"
+out=$BOOKIE_LOG_DIR/bookkeeper-$command-$HOSTNAME.out
+logfile=$BOOKIE_LOG_DIR/$BOOKIE_LOG_FILE
+
+rotate_out_log ()
+{
+  log=$1;
+  num=5;
+  if [ -n "$2" ]; then
+    num=$2
+  fi
+  if [ -f "$log" ]; then # rotate logs
+    while [ $num -gt 1 ]; do
+      prev=$(expr $num - 1)
+      [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
+      num=$prev
+    done
+    mv "$log" "$log.$num";
+  fi
+}
+
+mkdir -p "$BOOKIE_LOG_DIR"
+
+start()
+{
+  if [ -f $pid_file ]; then
+    PREVIOUS_PID=$(cat $pid_file)
+    if ps -p $PREVIOUS_PID > /dev/null 2>&1; then
+      echo $command running as process $PREVIOUS_PID.  Stop it first.
+      exit 1
+    fi
+  fi
+
+  rotate_out_log $out
+  echo starting $command, logging to $logfile
+  bookkeeper=$BK_HOME/bin/bookkeeper
+  nohup $bookkeeper $command "$@" > "$out" 2>&1 < /dev/null &
+  echo $! > $pid_file
+  sleep 1; head $out
+  sleep 2;
+  if ! ps -p $! > /dev/null ; then
+    exit 1
+  fi
+}
+
+stop()
+{
+  if [ -f $pid_file ]; then
+    TARGET_PID=$(cat $pid_file)
+    if ps -p $TARGET_PID > /dev/null 2>&1; then
+      echo stopping $command
+      kill $TARGET_PID
+
+      count=0
+      location=$BOOKIE_LOG_DIR
+      while ps -p $TARGET_PID > /dev/null 2>&1;
+      do
+        echo "Shutdown is in progress... Please wait..."
+        sleep 1
+        count=$(expr $count + 1)
+
+        if [ "$count" = "$BOOKIE_STOP_TIMEOUT" ]; then
+          break
+        fi
+      done
+
+      if [ "$count" != "$BOOKIE_STOP_TIMEOUT" ]; then
+        echo "Shutdown completed."
+      fi
+
+      if ps -p $TARGET_PID > /dev/null 2>&1; then
+        fileName=$location/$command.out
+        # Check for the java to use
+        if [[ -z ${JAVA_HOME} ]]; then
+          JSTACK=$(which jstack)
+          if [ $? -ne 0 ]; then
+            echo "Error: JAVA_HOME not set, and no jstack executable found in $PATH." 1>&2
+            exit 1
+          fi
+        else
+          JSTACK=${JAVA_HOME}/bin/jstack
+        fi
+        $JSTACK $TARGET_PID > $fileName
+        echo Thread dumps are taken for analysis at $fileName
+        if [ "$1" == "-force" ]
+        then
+          echo forcefully stopping $command
+          kill -9 $TARGET_PID >/dev/null 2>&1
+          echo Successfully stopped the process
+        else
+          echo "WARNNING :  Bookie Server is not stopped completely."
+          exit 1
+        fi
+      fi
+    else
+      echo no $command to stop
+    fi
+    rm $pid_file
+  else
+    echo no $command to stop
+  fi
+}
+case $startStop in
+  (start)
+    start "$*"
+    ;;
+
+  (stop)
+    stop $1
+    ;;
+  (restart)
+    forceStopFlag=$(echo "$*"|grep "\-force")
+    if [[ "$forceStopFlag" != "" ]]
+    then
+      stop "-force"
+    else
+      stop
+    fi
+    if [ "$?" == 0 ]
+    then
+      sleep 3
+      parameters="$*"
+      startParameters=${parameters//-force/}
+      start "$startParameters"
+    else
+      echo "WARNNING :  $command failed restart, for $command is not stopped completely."
+    fi
+    ;;
+  (*)
+    usage
+    echo $supportedargs
+    exit 1
+    ;;
+esac

bin/common.sh

@@ -0,0 +1,376 @@
+#!/usr/bin/env bash
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Check net.ipv6.bindv6only
+if [ -f /sbin/sysctl ] && [ -f /proc/sys/net/ipv6/bindv6only ]; then
+  # check if net.ipv6.bindv6only is set to 1
+  bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null)
+  if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ]
+  then
+    echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken"
+    echo "For more info (the following page also applies to bookkeeper): http://wiki.apache.org/hadoop/HadoopIPv6"
+    exit 1
+  fi
+fi
+
+# See the following page for extensive details on setting
+# up the JVM to accept JMX remote management:
+# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+# by default we allow local JMX connections
+if [ "x$JMXLOCALONLY" = "x" ]
+then
+  JMXLOCALONLY=false
+fi
+
+if [ "x$JMXDISABLE" = "x" ]
+then
+  # for some reason these two options are necessary on jdk6 on Ubuntu
+  #   accord to the docs they are not necessary, but otw jconsole cannot
+  #   do a local attach
+  JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY"
+else
+  echo "JMX disabled by user request" >&2
+fi
+
+# Check for the java to use
+if [[ -z ${JAVA_HOME} ]]; then
+  JAVA=$(which java)
+  if [ $? != 0 ]; then
+    echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2
+    exit 1
+  fi
+else
+  JAVA=${JAVA_HOME}/bin/java
+fi
+
+BINDIR=${BK_BINDIR:-"`dirname "$0"`"}
+BK_HOME=${BK_HOME:-"`cd ${BINDIR}/..;pwd`"}
+BK_CONFDIR=${BK_HOME}/conf
+DEFAULT_LOG_CONF=${BK_CONFDIR}/log4j2.xml
+
+source ${BK_CONFDIR}/nettyenv.sh
+source ${BK_CONFDIR}/bkenv.sh
+source ${BK_CONFDIR}/bk_cli_env.sh
+
+detect_jdk8() {
+  local is_java_8=$($JAVA -version 2>&1 | grep version | grep '"1\.8')
+  if [ -z "$is_java_8" ]; then
+     echo "0"
+  else
+     echo "1"
+  fi
+}
+
+# default netty settings
+NETTY_LEAK_DETECTION_LEVEL=${NETTY_LEAK_DETECTION_LEVEL:-"disabled"}
+
+USING_JDK8=$(detect_jdk8)
+
+if [ "$USING_JDK8" -ne "1" ]; then
+   DEFAULT_BOOKIE_GC_OPTS="-XX:+UseG1GC \
+    -XX:MaxGCPauseMillis=10 \
+    -XX:+ParallelRefProcEnabled \
+    -XX:+DisableExplicitGC"
+   DEFAULT_BOOKIE_GC_LOGGING_OPTS=""
+else
+  DEFAULT_BOOKIE_GC_OPTS="-XX:+UseG1GC \
+    -XX:MaxGCPauseMillis=10 \
+    -XX:+ParallelRefProcEnabled \
+    -XX:+UnlockExperimentalVMOptions \
+    -XX:+DoEscapeAnalysis \
+    -XX:ParallelGCThreads=32 \
+    -XX:ConcGCThreads=32 \
+    -XX:G1NewSizePercent=50 \
+    -XX:+DisableExplicitGC \
+    -XX:-ResizePLAB"
+  DEFAULT_BOOKIE_GC_LOGGING_OPTS="-XX:+PrintGCDetails \
+    -XX:+PrintGCApplicationStoppedTime  \
+    -XX:+UseGCLogFileRotation \
+    -XX:NumberOfGCLogFiles=5 \
+    -XX:GCLogFileSize=64m"
+fi
+
+BOOKIE_MAX_HEAP_MEMORY=${BOOKIE_MAX_HEAP_MEMORY:-"1g"}
+BOOKIE_MIN_HEAP_MEMORY=${BOOKIE_MIN_HEAP_MEMORY:-"1g"}
+BOOKIE_MAX_DIRECT_MEMORY=${BOOKIE_MAX_DIRECT_MEMORY:-"2g"}
+BOOKIE_MEM_OPTS=${BOOKIE_MEM_OPTS:-"-Xms${BOOKIE_MIN_HEAP_MEMORY} -Xmx${BOOKIE_MAX_HEAP_MEMORY} -XX:MaxDirectMemorySize=${BOOKIE_MAX_DIRECT_MEMORY}"}
+BOOKIE_GC_OPTS=${BOOKIE_GC_OPTS:-"${DEFAULT_BOOKIE_GC_OPTS}"}
+BOOKIE_GC_LOGGING_OPTS=${BOOKIE_GC_LOGGING_OPTS:-"${DEFAULT_BOOKIE_GC_LOGGING_OPTS}"}
+
+# default CLI JVM settings
+DEFAULT_CLI_GC_OPTS="-XX:+UseG1GC \
+    -XX:MaxGCPauseMillis=10"
+if [ "$USING_JDK8" -ne "1" ]; then
+  DEFAULT_CLI_GC_LOGGING_OPTS=""
+else
+  DEFAULT_CLI_GC_LOGGING_OPTS="-XX:+PrintGCDetails \
+    -XX:+PrintGCApplicationStoppedTime  \
+    -XX:+UseGCLogFileRotation \
+    -XX:NumberOfGCLogFiles=5 \
+    -XX:GCLogFileSize=64m"
+fi
+
+CLI_MAX_HEAP_MEMORY=${CLI_MAX_HEAP_MEMORY:-"512M"}
+CLI_MIN_HEAP_MEMORY=${CLI_MIN_HEAP_MEMORY:-"256M"}
+CLI_MEM_OPTS=${CLI_MEM_OPTS:-"-Xms${CLI_MIN_HEAP_MEMORY} -Xmx${CLI_MAX_HEAP_MEMORY}"}
+CLI_GC_OPTS=${CLI_GC_OPTS:-"${DEFAULT_CLI_GC_OPTS}"}
+CLI_GC_LOGGING_OPTS=${CLI_GC_LOGGING_OPTS:-"${DEFAULT_CLI_GC_LOGGING_OPTS}"}
+
+# module names
+BOOKIE_SERVER_MODULE_NAME="(org.apache.bookkeeper-)?bookkeeper-server"
+TABLE_SERVICE_MODULE_NAME="(org.apache.bookkeeper-)?stream-storage-server"
+
+is_released_binary() {
+  if [ -d ${BK_HOME}/lib ]; then
+    echo "true"
+    return
+  else
+    echo "false"
+    return
+  fi
+}
+
+find_module_jar_at() {
+  DIR=$1
+  MODULE=$2
+  REGEX="^${MODULE}-[0-9\\.]*((-[a-zA-Z]*(-[0-9]*)?)|(-SNAPSHOT))?.jar$"
+  if [ -d ${DIR} ]; then
+    cd ${DIR}
+    for f in *.jar; do
+      if [[ ${f} =~ ${REGEX} ]]; then
+        echo ${DIR}/${f}
+        return
+      fi
+    done
+  fi
+}
+
+find_module_release_jar() {
+  MODULE_NAME=$1
+  RELEASE_JAR=$(find_module_jar_at ${BK_HOME} ${MODULE_NAME})
+  if [ -n "${RELEASE_JAR}" ]; then
+    MODULE_JAR=${RELEASE_JAR}
+  else
+    RELEASE_JAR=$(find_module_jar_at ${BK_HOME}/lib ${MODULE_NAME})
+    if [ -n "${RELEASE_JAR}" ]; then
+      MODULE_JAR=${RELEASE_JAR}
+    fi
+  fi
+  echo ${RELEASE_JAR}
+  return
+}
+
+find_module_jar() {
+  MODULE_PATH=$1
+  MODULE_NAME=$2
+  RELEASE_JAR=$(find_module_jar_at ${BK_HOME} ${MODULE_NAME})
+  if [ -n "${RELEASE_JAR}" ]; then
+    MODULE_JAR=${RELEASE_JAR}
+  else
+    RELEASE_JAR=$(find_module_jar_at ${BK_HOME}/lib ${MODULE_NAME})
+    if [ -n "${RELEASE_JAR}" ]; then
+      MODULE_JAR=${RELEASE_JAR}
+    fi
+  fi
+
+  if [ -z "${MODULE_JAR}" ]; then
+    BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/target ${MODULE_NAME})
+    if [ -z "${BUILT_JAR}" ]; then
+      echo "Couldn't find module '${MODULE_NAME}' jar." >&2
+      read -p "Do you want me to run \`mvn install -DskipTests\` for you ? (y|n) " answer
+      case "${answer:0:1}" in
+        y|Y )
+          mkdir -p ${BK_HOME}/logs
+          output="${BK_HOME}/logs/build.out"
+          echo "see output at ${output} for the progress ..." >&2
+          mvn install -DskipTests &> ${output}
+          ;;
+        * )
+          exit 1
+          ;;
+      esac
+
+      BUILT_JAR=$(find_module_jar_at ${BK_HOME}/${MODULE_PATH}/target ${MODULE_NAME})
+    fi
+    if [ -n "${BUILT_JAR}" ]; then
+      MODULE_JAR=${BUILT_JAR}
+    fi
+  fi
+
+  if [ ! -e "${MODULE_JAR}" ]; then
+    echo "Could not find module '${MODULE_JAR}' jar." >&2
+    exit 1
+  fi
+  echo ${MODULE_JAR}
+  return
+}
+
+add_maven_deps_to_classpath() {
+  MODULE_PATH=$1
+  MVN="mvn"
+  if [ "$MAVEN_HOME" != "" ]; then
+    MVN=${MAVEN_HOME}/bin/mvn
+  fi
+
+  # Need to generate classpath from maven pom. This is costly so generate it
+  # and cache it. Save the file into our target dir so a mvn clean will get
+  # clean it up and force us create a new one.
+  f="${BK_HOME}/${MODULE_PATH}/target/cached_classpath.txt"
+  output="${BK_HOME}/${MODULE_PATH}/target/build_classpath.out"
+
+  if [ ! -f ${f} ]; then
+    echo "the classpath of module '${MODULE_PATH}' is not found, generating it ..." >&2
+    echo "see output at ${output} for the progress ..." >&2
+    ${MVN} -f "${BK_HOME}/${MODULE_PATH}/pom.xml" dependency:build-classpath -Dmdep.outputFile="target/cached_classpath.txt" &> ${output}
+    echo "the classpath of module '${MODULE_PATH}' is generated at '${f}'." >&2
+  fi
+}
+
+set_module_classpath() {
+  MODULE_PATH=$1
+  if [ -d "${BK_HOME}/lib" ]; then
+    BK_CLASSPATH=""
+    for i in ${BK_HOME}/lib/*.jar; do
+      BK_CLASSPATH=${BK_CLASSPATH}:${i}
+    done
+    echo ${BK_CLASSPATH}
+  else
+    add_maven_deps_to_classpath ${MODULE_PATH} >&2
+    cat ${BK_HOME}/${MODULE_PATH}/target/cached_classpath.txt
+  fi
+  return
+}
+
+build_bookie_jvm_opts() {
+  LOG_DIR=$1
+  GC_LOG_FILENAME=$2
+  if [ "$USING_JDK8" -eq "1" ]; then
+    echo "$BOOKIE_MEM_OPTS $BOOKIE_GC_OPTS $BOOKIE_GC_LOGGING_OPTS $BOOKIE_PERF_OPTS -Xloggc:${LOG_DIR}/${GC_LOG_FILENAME}"
+  else
+    echo "$BOOKIE_MEM_OPTS $BOOKIE_GC_OPTS $BOOKIE_GC_LOGGING_OPTS $BOOKIE_PERF_OPTS -Xlog:gc=info:file=${LOG_DIR}/${GC_LOG_FILENAME}::filecount=5,filesize=64m"
+  fi
+  return
+}
+
+build_cli_jvm_opts() {
+  LOG_DIR=$1
+  GC_LOG_FILENAME=$2
+  if [ "$USING_JDK8" -eq "1" ]; then
+    echo "$CLI_MEM_OPTS $CLI_GC_OPTS $CLI_GC_LOGGING_OPTS -Xloggc:${LOG_DIR}/${GC_LOG_FILENAME}"
+  else
+    echo "$CLI_MEM_OPTS $CLI_GC_OPTS $CLI_GC_LOGGING_OPTS -Xlog:gc=info:file=${LOG_DIR}/${GC_LOG_FILENAME}::filecount=5,filesize=64m"
+  fi
+  return
+}
+
+build_netty_opts() {
+  NETTY_OPTS="-Dio.netty.leakDetectionLevel=${NETTY_LEAK_DETECTION_LEVEL} -Dio.netty.tryReflectionSetAccessible=true"
+  # --add-opens does not exist on jdk8
+  if [ "$USING_JDK8" -eq "0" ]; then
+    # Enable java.nio.DirectByteBuffer
+    # https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java
+    # https://github.com/netty/netty/issues/12265
+    NETTY_OPTS="$NETTY_OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
+  fi
+  echo $NETTY_OPTS
+}
+
+build_logging_opts() {
+  CONF_FILE=$1
+  LOG_LEVEL=$2
+  LOG_APPENDER=$3
+  LOG_DIR=$4
+  LOG_FILE=$5
+
+  echo "-Dlog4j.configurationFile=`basename ${CONF_FILE}` \
+    -Dbookkeeper.log.root.level=${LOG_LEVEL} \
+    -Dbookkeeper.log.root.appender=${LOG_APPENDER} \
+    -Dbookkeeper.log.dir=${LOG_DIR} \
+    -Dbookkeeper.log.file=${LOG_FILE}"
+}
+
+build_cli_logging_opts() {
+  CONF_FILE=$1
+  LOG_LEVEL=$2
+  LOG_APPENDER=$3
+  LOG_DIR=$4
+  LOG_FILE=$5
+
+  echo "-Dlog4j.configurationFile=`basename ${CONF_FILE}` \
+    -Dbookkeeper.cli.log.root.level=${LOG_LEVEL} \
+    -Dbookkeeper.cli.log.root.appender=${LOG_APPENDER} \
+    -Dbookkeeper.cli.log.dir=${LOG_DIR} \
+    -Dbookkeeper.cli.log.file=${LOG_FILE}"
+}
+
+build_bookie_opts() {
+  BOOKIE_OPTS="-Djava.net.preferIPv4Stack=true"
+  # --add-opens does not exist on jdk8
+  if [ "$USING_JDK8" -eq "0" ]; then
+    # enable posix_fadvise usage in the Journal
+    BOOKIE_OPTS="$BOOKIE_OPTS --add-opens java.base/java.io=ALL-UNNAMED"
+    # DirectMemoryCRC32Digest
+    BOOKIE_OPTS="$BOOKIE_OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED"
+  fi
+  echo $BOOKIE_OPTS
+}
+
+find_table_service() {
+  BOOKIE_CONF_TO_CHECK=$1
+  SERVICE_COMMAND=$2
+
+  # check if it is a released binary
+  IS_RELEASED_BINARY=$(is_released_binary)
+
+  # check if table service is released
+  TABLE_SERVICE_RELEASED="true"
+  if [ "x${IS_RELEASED_BINARY}" == "xtrue" ]; then
+    TABLE_SERVICE_RELEASE_JAR=$(find_module_release_jar ${TABLE_SERVICE_MODULE_NAME})
+    if [ "x${TABLE_SERVICE_RELEASE_JAR}" == "x" ]; then
+      TABLE_SERVICE_RELEASED="false"
+    fi
+  fi
+
+  # check the configuration to see if table service is enabled or not.
+  if [ -z "${ENABLE_TABLE_SERVICE}" ]; then
+    # mask exit code if the configuration file doesn't contain `StreamStorageLifecycleComponent`
+    TABLE_SERVICE_SETTING=$(grep StreamStorageLifecycleComponent ${BOOKIE_CONF_TO_CHECK} | cat)
+    if [[ "${TABLE_SERVICE_SETTING}" =~ ^extraServerComponents.* ]]; then
+      if [ "x${TABLE_SERVICE_RELEASED}" == "xfalse" ]; then
+        echo "The release binary is built without table service. Please disable \`StreamStorageLifecycleComponent\` in your bookie configuration at '${BOOKIE_CONF_TO_CHECK}'."
+        return
+      fi
+      ENABLE_TABLE_SERVICE="true"
+    fi
+  fi
+
+  # standalone only run
+  if [ \( "x${SERVICE_COMMAND}" == "xstandalone" \) -a \( "x${TABLE_SERVICE_RELEASED}" == "xfalse" \) ]; then
+    echo "The release binary is built without table service. Use \`localbookie <n>\` instead of \`standalone\` for local development."
+    return
+  fi
+
+  if [ \( "x${SERVICE_COMMAND}" == "xstandalone" \) -o \( "x${ENABLE_TABLE_SERVICE}" == "xtrue" \) ]; then
+    echo "true"
+    return
+  else
+    echo "false"
+    return
+  fi
+}

bin/dlog

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+#
+# vim:et:ft=sh:sts=2:sw=2
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+BINDIR=`dirname "$0"`
+BK_HOME=`cd ${BINDIR}/..;pwd`
+
+source ${BK_HOME}/bin/common.sh
+
+DLOG_MODULE_NAME="(org.apache.distributedlog-)?distributedlog-core"
+DLOG_MODULE_PATH=stream/distributedlog/core
+DLOG_MODULE_HOME=${BK_HOME}/${DLOG_MODULE_PATH}
+
+# find the module jar
+DLOG_JAR=$(find_module_jar ${DLOG_MODULE_PATH} ${DLOG_MODULE_NAME})
+
+# set up the classpath
+DLOG_CLASSPATH=$(set_module_classpath ${DLOG_MODULE_PATH})
+
+# default variables
+DEFAULT_CONF=${BK_HOME}/conf/bk_server.conf
+DEFAULT_CLI_CONF=${BK_HOME}/conf/bk_server.conf
+DEFAULT_LOG_CONF=${BK_HOME}/conf/log4j2.xml
+DEFAULT_CLI_LOG_CONF=${BK_HOME}/conf/log4j2.cli.xml
+
+dlog_help() {
+    cat <<EOF
+Usage: dlog <command>
+where command is one of:
+    local               Run distributedlog sandbox
+    tool                Run distributedlog tool
+    admin               Run distributedlog admin tool
+    help                This help message
+
+or command is the full name of a class with a defined main() method.
+
+Environment variables:
+
+    BOOKIE_CONF            Bookie configuration file (default: ${DEFAULT_CONF})
+    BOOKIE_EXTRA_OPTS      Extra options to be passed to the bookie jvm
+    BOOKIE_EXTRA_CLASSPATH Add extra paths to the bookie classpath
+    CLI_CONF               CLI configuration file (default: ${DEFAULT_CLI_CONF})
+    CLI_EXTRA_OPTS         Extra options to be passed to the CLI jvm
+    CLI_EXTRA_CLASSPATH    Add extra paths to the CLI classpath
+
+These variable can also be set in conf/bkenv.sh & conf/bk_cli_env.sh
+EOF
+}
+
+# if no args specified, show usage
+if [ $# = 0 ]; then
+  dlog_help;
+  exit 1;
+fi
+
+# get arguments
+COMMAND=$1
+shift
+
+if [ ${COMMAND} == "local" ]; then
+  DLOG_CONF=${BOOKIE_CONF:-"${DEFAULT_CONF}"}
+  DLOG_LOG_CONF=${BOOKIE_LOG_CONF:-"${DEFAULT_LOG_CONF}"}
+  DLOG_LOG_DIR=${BOOKIE_LOG_DIR:-"${BK_HOME}/logs"}
+  DLOG_LOG_FILE=${BOOKIE_LOG_FILE:-"dlog.log"}
+  DLOG_ROOT_LOG_LEVEL=${BOOKIE_ROOT_LOG_LEVEL:-"INFO"}
+  DLOG_ROOT_LOG_APPENDER=${BOOKIE_ROOT_LOG_APPENDER:-"CONSOLE"}
+  DLOG_EXTRA_CLASSPATH=${BOOKIE_EXTRA_CLASSPATH}
+  DLOG_GC_OPTS=$(build_bookie_jvm_opts ${DLOG_LOG_DIR} "dlog_gc_%p.log")
+  DLOG_EXTRA_OPTS=${BOOKIE_EXTRA_OPTS}
+  DLOG_LOGGING_OPTS=$(build_logging_opts ${DLOG_LOG_CONF} ${DLOG_ROOT_LOG_LEVEL} ${DLOG_ROOT_LOG_APPENDER} ${DLOG_LOG_DIR} ${DLOG_LOG_FILE})
+else
+  DLOG_CONF=${CLI_CONF:-"${DEFAULT_CLI_CONF}"}
+  DLOG_LOG_CONF=${CLI_LOG_CONF:-"${DEFAULT_CLI_LOG_CONF}"}
+  DLOG_LOG_DIR=${CLI_LOG_DIR:-"${BK_HOME}/logs"}
+  DLOG_LOG_FILE=${CLI_LOG_FILE:-"dlog-cli.log"}
+  DLOG_ROOT_LOG_LEVEL=${CLI_ROOT_LOG_LEVEL:-"INFO"}
+  DLOG_ROOT_LOG_APPENDER=${CLI_ROOT_LOG_APPENDER:-"ROLLINGFILE"}
+  DLOG_EXTRA_CLASSPATH=${CLI_EXTRA_CLASSPATH}
+  DLOG_GC_OPTS=$(build_cli_jvm_opts ${DLOG_LOG_DIR} "dlog-cli-gc.log")
+  DLOG_EXTRA_OPTS=${CLI_EXTRA_OPTS}
+  DLOG_LOGGING_OPTS=$(build_cli_logging_opts ${DLOG_LOG_CONF} ${DLOG_ROOT_LOG_LEVEL} ${DLOG_ROOT_LOG_APPENDER} ${DLOG_LOG_DIR} ${DLOG_LOG_FILE})
+fi
+
+DLOG_CLASSPATH="$DLOG_JAR:$DLOG_CLASSPATH:$DLOG_EXTRA_CLASSPATH"
+DLOG_CLASSPATH="`dirname $DLOG_LOG_CONF`:$DLOG_CLASSPATH"
+
+# Build the OPTS
+BOOKIE_OPTS=$(build_bookie_opts)
+NETTY_OPTS=$(build_netty_opts)
+OPTS="${OPTS} -cp ${DLOG_CLASSPATH} ${BOOKIE_OPTS} ${DLOG_GC_OPTS} ${NETTY_OPTS} ${DLOG_LOGGING_OPTS} ${DLOG_EXTRA_OPTS}"
+
+if [ ! -d ${DLOG_LOG_DIR} ]; then
+    mkdir ${DLOG_LOG_DIR}
+fi
+
+#Change to BK_HOME to support relative paths
+cd "$BK_HOME"
+case "${COMMAND}" in
+  local)
+    exec "${JAVA}" ${OPTS} ${JMX_ARGS} -Dzookeeper.4lw.commands.whitelist='*' org.apache.distributedlog.LocalDLMEmulator $@
+    ;;
+  tool)
+    exec "${JAVA}" ${OPTS} org.apache.distributedlog.tools.Tool org.apache.distributedlog.tools.DistributedLogTool $@
+    ;;
+  admin)
+    exec "${JAVA}" ${OPTS} org.apache.distributedlog.tools.Tool org.apache.distributedlog.admin.DistributedLogAdmin $@
+    ;;
+  help)
+    dlog_help
+    ;;
+  *)
+    exec "${JAVA}" ${OPTS} ${COMMAND} $@
+    ;;
+esac
+

bin/standalone

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+#
+# vim:et:ft=sh:sts=2:sw=2
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+usage() {
+    cat <<EOF
+Usage: standalone <runtime> (up|down)
+where runtime is one of:
+    process          Run a standalone cluster in one process
+    docker-compose   Run a standalone cluster in a docker-composed cluster
+EOF
+}
+
+BINDIR=${BK_BINDIR:-"`dirname "$0"`"}
+
+if [ $# -lt 2 ]
+then
+  echo "Error: no enough arguments provided."
+  usage
+  exit 1
+fi
+
+runtime=$1
+shift
+action=$1
+shift
+
+case $runtime in
+  (process)
+    ${BINDIR}/standalone.process $action $@
+    ;;
+  (docker-compose)
+    DOCKER_COMPOSE=$(which docker-compose)
+    if [ $? != 0 ]; then
+      echo "Error: docker-compose is not found in ${PATH}." 1>&2
+      usage
+      exit 1
+    else
+      echo ${BINDIR}/standalone.docker-compose $action $@
+      ${BINDIR}/standalone.docker-compose $action $@
+    fi
+    ;;
+  (*)
+    echo "Error: unknown runtime $command"
+    usage
+    exit 1
+    ;;
+esac

bin/standalone.docker-compose

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+#
+# vim:et:ft=sh:sts=2:sw=2
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+BINDIR=${BK_BINDIR:-"`dirname "$0"`"}
+
+source ${BINDIR}/common.sh
+
+DOCKER_COMPOSE=docker-compose
+
+DATA_ROOT_DIR=${BK_DATA_DIR:-"${BK_HOME}/data"}
+mkdir -p ${DATA_ROOT_DIR}
+
+function gen_metadata_service_section() {
+  local cluster=$1
+  local image=$2
+  cat <<EOF
+  metadata-service:
+    image: ${image}
+    hostname: metadata-service
+    command: ["zookeeper"]
+    environment:
+      - ZK_dataDir=/data/zookeeper/data
+      - ZK_dataLogDir=/data/zookeeper/txlog
+      - ZK_standaloneEnabled=true
+    ports:
+      - "9990:9990"
+      - "2181:2181"
+    volumes:
+      - "${DATA_ROOT_DIR}/${cluster}/zookeeper/data:/data/zookeeper/data"
+      - "${DATA_ROOT_DIR}/${cluster}/zookeeper/txlog:/data/zookeeper/txlog"
+EOF
+}
+
+function gen_bookie_section() {
+  local cluster=$1
+  local bookie_name=$2
+  local bookie_port=$3
+  local bookie_http_port=$4
+  local bookie_grpc_port=$5
+  local image=$6
+  cat <<EOF
+  ${bookie_name}:
+    image: ${image}
+    depends_on:
+      - metadata-service
+    environment:
+      # metadata service
+      - BK_zkServers=metadata-service
+      - BK_zkLedgersRootPath=/ledgers
+      - BK_metadataServiceUri=zk://metadata-service/ledgers
+      # bookie
+      - BK_DATA_DIR=/data/bookkeeper
+      - BK_advertisedAddress=${bookie_name}
+      # bookie http
+      - BK_httpServerEnabled=true
+      # stream storage
+      - BK_extraServerComponents=org.apache.bookkeeper.stream.server.StreamStorageLifecycleComponent
+    ports:
+      - "${bookie_port}:3181"
+      - "${bookie_http_port}:8080"
+      - "${bookie_grpc_port}:4181"
+    volumes:
+      - "${DATA_ROOT_DIR}/${cluster}/${bookie_name}/journal:/data/bookkeeper/journal"
+      - "${DATA_ROOT_DIR}/${cluster}/${bookie_name}/ledgers:/data/bookkeeper/ledgers"
+EOF
+}
+
+function generate_docker_compose_file() {
+  local cluster=$1
+  local num_bookies=$2
+  local image=$3
+  local docker_compose_file="${DATA_ROOT_DIR}/${cluster}/docker-compose.yml"
+
+  local metadata_service_section=$(gen_metadata_service_section ${cluster} ${image})
+    
+  echo "version: '3'"                 >  ${docker_compose_file}
+  echo ""                             >> ${docker_compose_file}
+  echo "services:"                    >> ${docker_compose_file}
+  echo ""                             >> ${docker_compose_file}
+  echo "${metadata_service_section}"  >> ${docker_compose_file}
+  echo ""                             >> ${docker_compose_file}
+  local BI=0
+  while [ ${BI} -lt $((num_bookies)) ]; do
+    local bookie_port=$((3181 + BI))
+    local bookie_http_port=$((8080 + BI))
+    local bookie_grpc_port=$((4181 + BI))
+    local bookie_section=$(gen_bookie_section ${cluster} "bookie-${BI}" ${bookie_port} ${bookie_http_port} ${bookie_grpc_port} ${image})
+    echo "${bookie_section}"        >> ${docker_compose_file}
+    let BI=BI+1
+  done
+}
+
+function show_help() {
+  cat <<EOF
+Usage: standalone.docker-compose [-c <cluster_name>] [-h] <action:[up|down]>
+EOF
+}
+
+# main entrypoint
+
+CLUSTER_NAME="bk-standalone-dc"
+OPTIND=1
+IMAGE=${IMAGE:-"apachebookkeeper/bookkeeper-current"}
+NUM_BOOKIES=${NUM_BOOKIES:-"3"}
+
+while getopts "h:c:" opt; do
+  case "${opt}" in
+  c )
+    CLUSTER_NAME=${OPTARG}
+    echo "use cluster = '${CLUSTER_NAME}'."
+    ;;
+  h|\? )
+    show_help
+    exit 1
+    ;;
+  esac
+done
+
+shift $((OPTIND-1))
+
+[ "${1:-}" = "--" ] && shift
+
+if [ $# -le 0 ]; then
+  show_help
+  exit 1
+fi
+
+ACTION=$1
+DOCKER_COMPOSE_OPTS=""
+case "${ACTION}" in
+  up)
+    DOCKER_COMPOSE_OPTS="--detach"
+    ;;
+  down)
+    ;;
+  *)
+    echo "Unknown action : ${ACTION}"
+    show_help
+    exit 1
+    ;;
+esac
+
+CLUSTER_DATA_ROOT="${DATA_ROOT_DIR}/${CLUSTER_NAME}"
+mkdir -p ${CLUSTER_DATA_ROOT}
+
+# generate docker compose file 
+DOCKER_COMPOSE_FILE="${CLUSTER_DATA_ROOT}/docker-compose.yml"
+
+if [ ! -f ${DOCKER_COMPOSE_FILE} ]; then
+  generate_docker_compose_file ${CLUSTER_NAME} ${NUM_BOOKIES} ${IMAGE}
+fi
+
+cd ${CLUSTER_DATA_ROOT}
+
+
+
+${DOCKER_COMPOSE} $@ ${DOCKER_COMPOSE_OPTS}
+
+if [ $? == 0 -a "${ACTION}" == "up" ]; then
+    echo ""
+    echo "Standalone cluster '${CLUSTER_NAME}' is up running."
+    echo "Use following uris to connect to standalone cluster:"
+    echo ""
+    echo "  - metadata service uri = 'zk://localhost/ledgers'"
+    echo "  - dlog uri             = 'distributedlog://localhost/distributedlog'"
+fi
+exit $?