Merge remote-tracking branch 'origin/spill-disk' into spill-disk

.github/ISSUE_TEMPLATE/20_bug_report.yml

@@ -8,17 +8,17 @@ body:
       value: |
         Thank you very much for submitting feedback to Databend to help Databend develop better.
 
-        If it is an idea or help wanted, please go to: [Discussion](https://github.com/datafuselabs/databend/discussions)
+        If it is an idea or help wanted, please go to: [Discussion](https://github.com/databendlabs/databend/discussions)
 
   - type: checkboxes
     attributes:
       label: Search before asking
       description: >
-        Please make sure to search in the [issues](https://github.com/datafuselabs/databend/issues) first to see
+        Please make sure to search in the [issues](https://github.com/databendlabs/databend/issues) first to see
         whether the same issue was reported already.
       options:
         - label: >
-            I had searched in the [issues](https://github.com/datafuselabs/databend/issues) and found no similar
+            I had searched in the [issues](https://github.com/databendlabs/databend/issues) and found no similar
             issues.
           required: true
 

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,5 @@
 blank_issues_enabled: true
 contact_links:
   - name: Question
-    url: https://github.com/datafuselabs/databend/discussions/categories/q-a
+    url: https://github.com/databendlabs/databend/discussions/categories/q-a
     about: Please ask and answer questions on the discussions Q&A category.

.github/actions/publish_binary/action.yml

@@ -44,7 +44,7 @@ runs:
       if: inputs.category == 'default'
       run: |
         aws s3 cp ${{ steps.name.outputs.name }}.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}.tar.gz --no-progress
-        gh api /repos/datafuselabs/databend/tags > tags.json
+        gh api /repos/databendlabs/databend/tags > tags.json
         aws s3 cp ./tags.json s3://repo/databend/tags.json
-        gh api /repos/datafuselabs/databend/releases > releases.json
+        gh api /repos/databendlabs/databend/releases > releases.json
         aws s3 cp ./releases.json s3://repo/databend/releases.json

.github/actions/publish_deb/action.yml

@@ -32,19 +32,19 @@ runs:
         version=${{ inputs.version }}
         deb_version=${version/-/.}
         deb_version=${deb_version/v/}
-        wget -q https://github.com/datafuselabs/databend/releases/download/${version}/databend_${deb_version}_amd64.deb
-        wget -q https://github.com/datafuselabs/databend/releases/download/${version}/databend_${deb_version}_arm64.deb
+        wget -q https://github.com/databendlabs/databend/releases/download/${version}/databend_${deb_version}_amd64.deb
+        wget -q https://github.com/databendlabs/databend/releases/download/${version}/databend_${deb_version}_arm64.deb
         reprepro includedeb stable databend_${deb_version}_amd64.deb
         reprepro includedeb stable databend_${deb_version}_arm64.deb
 
     - name: Add BendSQL
       shell: bash
       working-directory: scripts/distribution/deb
       run: |
-        version=$(gh release view --repo datafuselabs/bendsql --json name | jq -r '.name')
+        version=$(gh release view --repo databendlabs/bendsql --json name | jq -r '.name')
         deb_version=${version/v/}
-        wget -q https://github.com/datafuselabs/bendsql/releases/download/${version}/bendsql_${deb_version}_amd64.deb
-        wget -q https://github.com/datafuselabs/bendsql/releases/download/${version}/bendsql_${deb_version}_arm64.deb
+        wget -q https://github.com/databendlabs/bendsql/releases/download/${version}/bendsql_${deb_version}_amd64.deb
+        wget -q https://github.com/databendlabs/bendsql/releases/download/${version}/bendsql_${deb_version}_arm64.deb
         reprepro includedeb stable bendsql_${deb_version}_amd64.deb
         reprepro includedeb stable bendsql_${deb_version}_arm64.deb
 

.github/actions/setup_bendsql/action.yml

@@ -10,7 +10,7 @@ runs:
         if bendsql --version; then
           exit 0
         fi
-        curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/datafuselabs/bendsql/releases/download/v0.18.3/bendsql-x86_64-unknown-linux-gnu.tar.gz
+        curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/databendlabs/bendsql/releases/download/v0.18.3/bendsql-x86_64-unknown-linux-gnu.tar.gz
         tar -xzf /tmp/bendsql.tar.gz -C /tmp
         mv /tmp/bendsql /usr/local/bin/bendsql
         bendsql --version
@@ -21,7 +21,7 @@ runs:
         if bendsql --version; then
           exit 0
         fi
-        curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/datafuselabs/bendsql/releases/download/v0.18.3/bendsql-x86_64-apple-darwin.tar.gz
+        curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/databendlabs/bendsql/releases/download/v0.18.3/bendsql-x86_64-apple-darwin.tar.gz
         tar -xzf /tmp/bendsql.tar.gz -C /tmp
         mv /tmp/bendsql /usr/local/bin/bendsql
         bendsql --version

.github/scripts/notify_release.js

@@ -43,7 +43,7 @@ module.exports = async ({ context, core }) => {
               {
                 tag: "a",
                 text: "Release Notes",
-                href: `https://github.com/datafuselabs/databend/releases/tag/${VERSION}`,
+                href: `https://github.com/databendlabs/databend/releases/tag/${VERSION}`,
               },
             ],
           ],

.github/workflows/release.yml

@@ -71,7 +71,7 @@ jobs:
       - name: Checkout Docs
         uses: actions/checkout@v4
         with:
-          repository: datafuselabs/databend-docs
+          repository: databendlabs/databend-docs
           ref: main
       - name: Get date
         shell: bash
@@ -83,10 +83,10 @@ jobs:
           mkdir -p docs/release-stable
           df="docs/release-stable/${{ env.DATE }}_${{ needs.create_release.outputs.version }}.md"
           echo "---" > $df
-          gh release view --repo datafuselabs/databend ${{ needs.create_release.outputs.version }} >> $df
+          gh release view --repo databendlabs/databend ${{ needs.create_release.outputs.version }} >> $df
           sed -i -E 's/^--$/---/g' $df
           sed -i -E '/^asset:/d' $df
-          sed -i -E 's_https://github.com/datafuselabs/databend/pull/([0-9]+)_[#\1](https://github.com/datafuselabs/databend/pull/\1)_g' $df
+          sed -i -E 's_https://github.com/databendlabs/databend/pull/([0-9]+)_[#\1](https://github.com/databendlabs/databend/pull/\1)_g' $df
           git add docs/release-stable
           git status
       - uses: peter-evans/create-pull-request@v4
@@ -649,7 +649,7 @@ jobs:
   #     - name: checkout share endpoint
   #       uses: actions/checkout@v4
   #       with:
-  #         repository: datafuselabs/share-endpoint
+  #         repository: databendlabs/share-endpoint
   #         token: ${{ secrets.DATABEND_BOT_TOKEN }}
   #         path: share-endpoint
   #     - name: Download artifacts

Cargo.toml

@@ -409,7 +409,7 @@ openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev =
 orc-rust = { git = "https://github.com/datafuse-extras/datafusion-orc", rev = "03372b97" }
 recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }
-tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "37aeac0" }
-tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "37aeac0", package = "tantivy-common" }
-tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "124a8fc" }
+tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370" }
+tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370", package = "tantivy-common" }
+tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "0e300e9" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }

src/common/arrow/src/arrow/array/list/mutable.rs

@@ -182,7 +182,9 @@ impl<O: Offset, M: MutableArray> MutableListArray<O, M> {
     pub fn try_push_valid(&mut self) -> Result<()> {
         let total_length = self.values.len();
         let offset = self.offsets.last().to_usize();
-        let length = total_length.checked_sub(offset).ok_or(Error::Overflow)?;
+        let length = total_length
+            .checked_sub(offset)
+            .ok_or_else(|| Error::Overflow)?;
 
         self.offsets.try_push_usize(length)?;
         if let Some(validity) = &mut self.validity {

src/common/arrow/src/arrow/offset.rs

@@ -127,7 +127,9 @@ impl<O: Offset> Offsets<O> {
     pub fn try_push(&mut self, length: O) -> Result<(), Error> {
         let old_length = self.last();
         assert!(length >= O::zero());
-        let new_length = old_length.checked_add(&length).ok_or(Error::Overflow)?;
+        let new_length = old_length
+            .checked_add(&length)
+            .ok_or_else(|| Error::Overflow)?;
         self.0.push(new_length);
         Ok(())
     }
@@ -140,10 +142,12 @@ impl<O: Offset> Offsets<O> {
     /// * checks that this length does not overflow
     #[inline]
     pub fn try_push_usize(&mut self, length: usize) -> Result<(), Error> {
-        let length = O::from_usize(length).ok_or(Error::Overflow)?;
+        let length = O::from_usize(length).ok_or_else(|| Error::Overflow)?;
 
         let old_length = self.last();
-        let new_length = old_length.checked_add(&length).ok_or(Error::Overflow)?;
+        let new_length = old_length
+            .checked_add(&length)
+            .ok_or_else(|| Error::Overflow)?;
         self.0.push(new_length);
         Ok(())
     }
@@ -267,8 +271,8 @@ impl<O: Offset> Offsets<O> {
 
         let last_offset = original_offset
             .checked_add(total_length)
-            .ok_or(Error::Overflow)?;
-        O::from_usize(last_offset).ok_or(Error::Overflow)?;
+            .ok_or_else(|| Error::Overflow)?;
+        O::from_usize(last_offset).ok_or_else(|| Error::Overflow)?;
         Ok(())
     }
 
@@ -279,7 +283,9 @@ impl<O: Offset> Offsets<O> {
         let mut length = *self.last();
         let other_length = *other.last();
         // check if the operation would overflow
-        length.checked_add(&other_length).ok_or(Error::Overflow)?;
+        length
+            .checked_add(&other_length)
+            .ok_or_else(|| Error::Overflow)?;
 
         let lengths = other.as_slice().windows(2).map(|w| w[1] - w[0]);
         let offsets = lengths.map(|new_length| {
@@ -306,7 +312,9 @@ impl<O: Offset> Offsets<O> {
         let other_length = other.last().expect("Length to be non-zero");
         let mut length = *self.last();
         // check if the operation would overflow
-        length.checked_add(other_length).ok_or(Error::Overflow)?;
+        length
+            .checked_add(other_length)
+            .ok_or_else(|| Error::Overflow)?;
 
         let lengths = other.windows(2).map(|w| w[1] - w[0]);
         let offsets = lengths.map(|new_length| {

src/common/base/src/base/string.rs

@@ -193,8 +193,7 @@ pub fn mask_connection_info(sql: &str) -> String {
 /// Maximum length of the SQL query to be displayed or log.
 /// If the query exceeds this length and starts with keywords,
 /// it will be truncated and appended with the remaining length.
-pub fn short_sql(sql: String) -> String {
-    const MAX_LENGTH: usize = 128;
+pub fn short_sql(sql: String, max_length: u64) -> String {
     let keywords = ["INSERT"];
 
     fn starts_with_any(query: &str, keywords: &[&str]) -> bool {
@@ -209,10 +208,10 @@ pub fn short_sql(sql: String) -> String {
     // of multiple Unicode code points.
     // This ensures that we handle complex characters like emojis or
     // accented characters properly.
-    if query.graphemes(true).count() > MAX_LENGTH && starts_with_any(query, &keywords) {
-        let truncated: String = query.graphemes(true).take(MAX_LENGTH).collect();
+    if query.graphemes(true).count() > max_length as usize && starts_with_any(query, &keywords) {
+        let truncated: String = query.graphemes(true).take(max_length as usize).collect();
         let original_length = query.graphemes(true).count();
-        let remaining_length = original_length.saturating_sub(MAX_LENGTH);
+        let remaining_length = original_length.saturating_sub(max_length as usize);
         // Append the remaining length indicator
         truncated + &format!("...[{} more characters]", remaining_length)
     } else {

src/common/base/src/headers.rs

@@ -25,6 +25,7 @@ pub const HEADER_NODE_ID: &str = "X-DATABEND-NODE-ID";
 
 pub const HEADER_QUERY_STATE: &str = "X-DATABEND-QUERY-STATE";
 pub const HEADER_QUERY_PAGE_ROWS: &str = "X-DATABEND-QUERY-PAGE-ROWS";
+pub const HEADER_VERSION: &str = "X-DATABEND-VERSION";
 
 pub const HEADER_SIGNATURE: &str = "X-DATABEND-SIGNATURE";
 pub const HEADER_AUTH_METHOD: &str = "X-DATABEND-AUTH-METHOD";

src/common/base/src/lib.rs

@@ -34,6 +34,7 @@ pub mod headers;
 pub mod mem_allocator;
 pub mod rangemap;
 pub mod runtime;
+pub mod vec_ext;
 pub mod version;
 
 pub use runtime::dump_backtrace;

src/common/base/src/vec_ext.rs

@@ -0,0 +1,27 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub trait VecExt<T> {
+    /// Remove the first element that is equal to the given item.
+    fn remove_first(&mut self, item: &T) -> Option<T>
+    where T: PartialEq;
+}
+
+impl<T> VecExt<T> for Vec<T> {
+    fn remove_first(&mut self, item: &T) -> Option<T>
+    where T: PartialEq {
+        let pos = self.iter().position(|x| x == item)?;
+        Some(self.remove(pos))
+    }
+}