From be7c2b9bc19499e315b178637a6a0163fbe98b25 Mon Sep 17 00:00:00 2001 From: Adi Eldar Date: Mon, 27 Jul 2026 00:10:53 +0300 Subject: [PATCH] Update slm_embeddings_fl documentation (#7559) * Update slm_embeddings_fl documentation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Apply authoring assistant suggestions Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../functions-library/slm-embeddings-fl.md | 2 +- data-explorer/kusto/functions-library/toc.yml | 2 +- .../kusto/includes/slm-embeddings-fl-adx.md | 67 ++++--- .../includes/slm-embeddings-fl-fabric.md | 172 +++++++++--------- 4 files changed, 128 insertions(+), 115 deletions(-) diff --git a/data-explorer/kusto/functions-library/slm-embeddings-fl.md b/data-explorer/kusto/functions-library/slm-embeddings-fl.md index e74bc0c61b..57749be0f0 100644 --- a/data-explorer/kusto/functions-library/slm-embeddings-fl.md +++ b/data-explorer/kusto/functions-library/slm-embeddings-fl.md @@ -3,7 +3,7 @@ title: slm_embeddings_fl() description: This article describes the slm_embeddings_fl() user-defined function. ms.reviewer: adieldar ms.topic: reference -ms.date: 12/16/2025 +ms.date: 07/26/2026 monikerRange: "microsoft-fabric || azure-data-explorer" --- # slm_embeddings_fl() diff --git a/data-explorer/kusto/functions-library/toc.yml b/data-explorer/kusto/functions-library/toc.yml index 25cf80c6f4..7db4e4d800 100644 --- a/data-explorer/kusto/functions-library/toc.yml +++ b/data-explorer/kusto/functions-library/toc.yml @@ -183,7 +183,7 @@ items: displayName: functions library, anomaly detection, univariate, change point href: series-uv-change-points-fl.md - name: slm_embeddings_fl() - displayName: functions library, text embeddings, SLM, vector embeddings, text analytics, NLP, AI + displayName: functions library, text embeddings, SLM, vector embeddings, text analytics, NLP, AI, SecBERT, cybersecurity href: slm-embeddings-fl.md - name: time_weighted_avg_fl() displayName: functions library, binning, time weighted, time series, interpolation diff --git a/data-explorer/kusto/includes/slm-embeddings-fl-adx.md b/data-explorer/kusto/includes/slm-embeddings-fl-adx.md index acfa4146ec..02026ccf42 100644 --- a/data-explorer/kusto/includes/slm-embeddings-fl-adx.md +++ b/data-explorer/kusto/includes/slm-embeddings-fl-adx.md @@ -1,10 +1,10 @@ --- ms.topic: include -ms.date: 05/07/2026 +ms.date: 07/26/2026 --- The function `slm_embeddings_fl()` is a [UDF (user-defined function)](../query/functions/user-defined-functions.md) that generates text embeddings using local Small Language Models (SLM). This function converts text into numerical vector representations that can be used for semantic search, similarity analysis, and other natural language processing tasks. -Currently the function supports [harrier-v1-270m](https://huggingface.co/microsoft/harrier-oss-v1-270m), [jina-v2-small](https://huggingface.co/jinaai/jina-embeddings-v2-small-en), and [e5-small-v2](https://huggingface.co/intfloat/e5-small-v2) models. +Currently, the function supports [harrier-v1-270m](https://huggingface.co/microsoft/harrier-oss-v1-270m), [jina-v2-small](https://huggingface.co/jinaai/jina-embeddings-v2-small-en), [e5-small-v2](https://huggingface.co/intfloat/e5-small-v2), and [SecBERT](https://huggingface.co/jackaduma/SecBERT) models. SecBERT is designed for cybersecurity text and generates 768-dimensional embeddings. ## Prerequisites @@ -30,13 +30,14 @@ Note that this change requires [AllDatabasesAdmin](../access-control/role-based- |*text_col*| `string` | :heavy_check_mark:|The name of the column containing the text to embed.| |*embeddings_col*| `string` | :heavy_check_mark:|The name of the column to store the output embeddings.| |*batch_size*| `int` ||The number of texts to process in each batch. Default is 32.| -|*model_name*| `string` ||The name of the embedding model to use. Supported values are `harrier-v1-270m` (default), `jina-v2-small`, and `e5-small-v2`.| -|*prefix*| `string` ||The text prefix to add before each input. Default is `query:`. For the Harrier and E5 models, use `query:` for search queries and `passage:` for documents to be searched (for the Harrier model, `passage:` maps to the empty task). This parameter is ignored for the Jina model.| +|*model_name*| `string` ||The name of the embedding model to use. Supported values are `harrier-v1-270m` (default), `jina-v2-small`, `e5-small-v2`, and `secbert`. The value must match the model artifact ZIP file name without the `.zip` extension.| +|*prefix*| `string` ||The text prefix to add before each input. Default is `query:`. For the Harrier and E5 models, use `query:` for search queries and `passage:` for documents to be searched (for the Harrier model, `passage:` maps to the empty task). This parameter is ignored for the Jina and SecBERT models.| ## Function definition -* You can define the function by either embedding its code as a query-defined function, or creating it as a stored function in your database. -* To optimize storage and latency you can delete external artifacts for models that are not used. +* Define the function by either embedding its code as a query-defined function or creating it as a stored function in your database. +* Each invocation loads the embedding engine and only the selected model artifact. To optimize storage, delete external artifacts for models that aren't used. +* If the Python sandbox image doesn't include the `tokenizers` package, uncomment both `tokenizers-0.22.1.whl` lines in the function definition. ### [Query-defined](#tab/query-defined) @@ -48,11 +49,13 @@ Define the function using the following [let statement](../query/let-statement.m ~~~kusto let slm_embeddings_fl = (tbl:(*), text_col:string, embeddings_col:string, batch_size:int=32, model_name:string='harrier-v1-270m', prefix:string='query:') { + let artifact_root = 'https://artifactswestus.z22.web.core.windows.net/models/SLM/'; + let engine_artifact = 'embedding_engine.zip'; let kwargs = bag_pack('text_col', text_col, 'embeddings_col', embeddings_col, 'batch_size', batch_size, 'model_name', model_name, 'prefix', prefix); let code = ```if 1: from sandbox_utils import Zipackage Zipackage.install('embedding_engine.zip') - Zipackage.install('tokenizers-0.22.1.whl') # redundant if tokenizers package is included in the Python image +# Zipackage.install('tokenizers-0.22.1.whl') # redundant if tokenizers package is included in the Python image from embedding_factory import create_embedding_engine @@ -65,18 +68,18 @@ let slm_embeddings_fl = (tbl:(*), text_col:string, embeddings_col:string, batch_ Zipackage.install(f'{model_name}.zip') engine = create_embedding_engine(model_name, cache_dir="C:\\Temp") - embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used only for E5 + embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used by E5 and Harrier; Jina and SecBERT ignore it result = df result[embeddings_col] = list(embeddings) ```; tbl - | evaluate hint.distribution=per_node python(typeof(*), code, kwargs, external_artifacts = bag_pack( - 'embedding_engine.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/embedding_engine.zip', - 'tokenizers-0.22.1.whl', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/tokenizers-0.22.1-cp39-abi3-win_amd64.whl', - 'harrier-v1-270m.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/harrier-v1-270m.zip', - 'jina-v2-small.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/jina-v2-small.zip', - 'e5-small-v2.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/e5-small-v2.zip')) + | evaluate hint.distribution=per_node python( + typeof(*), code, kwargs, + external_artifacts=bag_pack( +// 'tokenizers-0.22.1.whl', strcat(artifact_root, 'tokenizers-0.22.1-cp39-abi3-win_amd64.whl'), + 'embedding_engine.zip', strcat(artifact_root, engine_artifact), + strcat(model_name, '.zip'), strcat(artifact_root, model_name, '.zip'))) }; // Write your query to use the function here. ~~~ @@ -92,11 +95,13 @@ Define the stored function once using the following [`.create function`](../mana .create-or-alter function with (folder = "Packages\\AI", docstring = "Embedding using local SLM") slm_embeddings_fl(tbl:(*), text_col:string, embeddings_col:string, batch_size:int=32, model_name:string='harrier-v1-270m', prefix:string='query:') { + let artifact_root = 'https://artifactswestus.z22.web.core.windows.net/models/SLM/'; + let engine_artifact = 'embedding_engine.zip'; let kwargs = bag_pack('text_col', text_col, 'embeddings_col', embeddings_col, 'batch_size', batch_size, 'model_name', model_name, 'prefix', prefix); let code = ```if 1: from sandbox_utils import Zipackage Zipackage.install('embedding_engine.zip') - Zipackage.install('tokenizers-0.22.1.whl') # redundant if tokenizers package is included in the Python image +# Zipackage.install('tokenizers-0.22.1.whl') # redundant if tokenizers package is included in the Python image from embedding_factory import create_embedding_engine @@ -109,18 +114,18 @@ slm_embeddings_fl(tbl:(*), text_col:string, embeddings_col:string, batch_size:in Zipackage.install(f'{model_name}.zip') engine = create_embedding_engine(model_name, cache_dir="C:\\Temp") - embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used only for E5 + embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used by E5 and Harrier; Jina and SecBERT ignore it result = df result[embeddings_col] = list(embeddings) ```; tbl - | evaluate hint.distribution=per_node python(typeof(*), code, kwargs, external_artifacts = bag_pack( - 'embedding_engine.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/embedding_engine.zip', - 'tokenizers-0.22.1.whl', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/tokenizers-0.22.1-cp39-abi3-win_amd64.whl', - 'harrier-v1-270m.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/harrier-v1-270m.zip', - 'jina-v2-small.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/jina-v2-small.zip', - 'e5-small-v2.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/e5-small-v2.zip')) + | evaluate hint.distribution=per_node python( + typeof(*), code, kwargs, + external_artifacts=bag_pack( +// 'tokenizers-0.22.1.whl', strcat(artifact_root, 'tokenizers-0.22.1-cp39-abi3-win_amd64.whl'), + 'embedding_engine.zip', strcat(artifact_root, engine_artifact), + strcat(model_name, '.zip'), strcat(artifact_root, model_name, '.zip'))) } ~~~ @@ -139,11 +144,13 @@ To use a query-defined function, invoke it after the embedded function definitio ~~~kusto let slm_embeddings_fl=(tbl:(*), text_col:string, embeddings_col:string, batch_size:int=32, model_name:string='harrier-v1-270m', prefix:string='query:') { + let artifact_root = 'https://artifactswestus.z22.web.core.windows.net/models/SLM/'; + let engine_artifact = 'embedding_engine.zip'; let kwargs = bag_pack('text_col', text_col, 'embeddings_col', embeddings_col, 'batch_size', batch_size, 'model_name', model_name, 'prefix', prefix); let code = ```if 1: from sandbox_utils import Zipackage Zipackage.install('embedding_engine.zip') - Zipackage.install('tokenizers-0.22.1.whl') # redundant if tokenizers package is included in the Python image +# Zipackage.install('tokenizers-0.22.1.whl') # redundant if tokenizers package is included in the Python image from embedding_factory import create_embedding_engine @@ -156,18 +163,18 @@ let slm_embeddings_fl=(tbl:(*), text_col:string, embeddings_col:string, batch_si Zipackage.install(f'{model_name}.zip') engine = create_embedding_engine(model_name, cache_dir="C:\\Temp") - embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used only for E5 + embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used by E5 and Harrier; Jina and SecBERT ignore it result = df result[embeddings_col] = list(embeddings) ```; tbl - | evaluate hint.distribution=per_node python(typeof(*), code, kwargs, external_artifacts = bag_pack( - 'embedding_engine.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/embedding_engine.zip', - 'tokenizers-0.22.1.whl', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/tokenizers-0.22.1-cp39-abi3-win_amd64.whl', - 'harrier-v1-270m.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/harrier-v1-270m.zip', - 'jina-v2-small.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/jina-v2-small.zip', - 'e5-small-v2.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/e5-small-v2.zip')) + | evaluate hint.distribution=per_node python( + typeof(*), code, kwargs, + external_artifacts=bag_pack( +// 'tokenizers-0.22.1.whl', strcat(artifact_root, 'tokenizers-0.22.1-cp39-abi3-win_amd64.whl'), + 'embedding_engine.zip', strcat(artifact_root, engine_artifact), + strcat(model_name, '.zip'), strcat(artifact_root, model_name, '.zip'))) }; // // Create a sample dataset with text passages diff --git a/data-explorer/kusto/includes/slm-embeddings-fl-fabric.md b/data-explorer/kusto/includes/slm-embeddings-fl-fabric.md index d1ce183b99..410a0209dc 100644 --- a/data-explorer/kusto/includes/slm-embeddings-fl-fabric.md +++ b/data-explorer/kusto/includes/slm-embeddings-fl-fabric.md @@ -1,10 +1,10 @@ --- ms.topic: include -ms.date: 05/07/2026 +ms.date: 07/26/2026 --- The function `slm_embeddings_fl()` is a [UDF (user-defined function)](../query/functions/user-defined-functions.md) that generates text embeddings using local Small Language Models (SLM). This function converts text into numerical vector representations that can be used for semantic search, similarity analysis, and other natural language processing tasks. -Currently the function supports [harrier-v1-270m](https://huggingface.co/microsoft/harrier-oss-v1-270m), [jina-v2-small](https://huggingface.co/jinaai/jina-embeddings-v2-small-en), and [e5-small-v2](https://huggingface.co/intfloat/e5-small-v2) models. +Currently, the function supports [harrier-v1-270m](https://huggingface.co/microsoft/harrier-oss-v1-270m), [jina-v2-small](https://huggingface.co/jinaai/jina-embeddings-v2-small-en), [e5-small-v2](https://huggingface.co/intfloat/e5-small-v2), and [SecBERT](https://huggingface.co/jackaduma/SecBERT) models. SecBERT is designed for cybersecurity text and generates 768-dimensional embeddings. ## Prerequisites @@ -24,15 +24,15 @@ Currently the function supports [harrier-v1-270m](https://huggingface.co/microso |*text_col*| `string` | :heavy_check_mark:|The name of the column containing the text to embed.| |*embeddings_col*| `string` | :heavy_check_mark:|The name of the column to store the output embeddings.| |*batch_size*| `int` ||The number of texts to process in each batch. Default is 32.| -|*model_name*| `string` ||The name of the embedding model to use. Supported values are `harrier-v1-270m` (default), `jina-v2-small`, and `e5-small-v2`.| -|*prefix*| `string` ||The text prefix to add before each input. Default is `query:`. For the Harrier and E5 models, use `query:` for search queries and `passage:` for documents to be searched (for the Harrier model, `passage:` maps to the empty task). This parameter is ignored for the Jina model.| +|*model_name*| `string` ||The name of the embedding model to use. Supported values are `harrier-v1-270m` (default), `jina-v2-small`, `e5-small-v2`, and `secbert`. The value must match the model artifact ZIP file name without the `.zip` extension.| +|*prefix*| `string` ||The text prefix to add before each input. Default is `query:`. For the Harrier and E5 models, use `query:` for search queries and `passage:` for documents to be searched (for the Harrier model, `passage:` maps to the empty task). This parameter is ignored for the Jina and SecBERT models.| ## Function definition -* Download the artifacts in the KQL code below (at the end of the code block see the external_artifacts parameter that reference artifacts, e.g https://artifactswestus.z22.web.core.windows.net/models/SLM/embedding_engine.zip) and upload them to your lakehouse. -* In the KQL code below update the artifacts paths to their one lake paths (e.g. https://msit-onelake.dfs.fabric.microsoft.com/MY_WORKSPACE/MY_LAKEHOUSE.Lakehouse/Files/models/SLM/embedding_engine.zip). -* You can define the function by either embedding its code as a query-defined function, or creating it as a stored function in your database. -* To optimize storage and latency you can delete external artifacts for models that aren't used. +* Download `embedding_engine.zip` and the model ZIP files you plan to use, and upload them to your lakehouse. +* In the KQL code, update `artifact_root` to the OneLake folder that contains the artifacts, for example `https://msit-onelake.dfs.fabric.microsoft.com/MY_WORKSPACE/MY_LAKEHOUSE.Lakehouse/Files/models/SLM/`. +* Define the function by either embedding its code as a query-defined function or creating it as a stored function in your database. +* Each invocation loads the embedding engine and only the selected model artifact. To optimize storage, delete external artifacts for models that aren't used. ### [Query-defined](#tab/query-defined) @@ -44,35 +44,37 @@ Define the function using the following [let statement](../query/let-statement.m ~~~kusto let slm_embeddings_fl = (tbl:(*), text_col:string, embeddings_col:string, batch_size:int=32, model_name:string='harrier-v1-270m', prefix:string='query:') { + let artifact_root = 'https://msit-onelake.dfs.fabric.microsoft.com/MY_WORKSPACE/MY_LAKEHOUSE.Lakehouse/Files/models/SLM/'; + let engine_artifact = 'embedding_engine.zip'; let kwargs = bag_pack('text_col', text_col, 'embeddings_col', embeddings_col, 'batch_size', batch_size, 'model_name', model_name, 'prefix', prefix); let code = ```if 1: - import os - from sandbox_utils import Zipackage - Zipackage.install('embedding_engine.zip') - - from embedding_factory import create_embedding_engine - - text_col = kargs["text_col"] - embeddings_col = kargs["embeddings_col"] - batch_size = kargs["batch_size"] - model_name = kargs["model_name"] - prefix = kargs["prefix"] - - Zipackage.install(f'{model_name}.zip') - - work_dir = os.environ.get("UPLOAD_PATH") # "/app/temp" - engine = create_embedding_engine(model_name, cache_dir=work_dir) - embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used only for E5 - - result = df - result[embeddings_col] = list(embeddings) + import os + from sandbox_utils import Zipackage + Zipackage.install('embedding_engine.zip') + + from embedding_factory import create_embedding_engine + + text_col = kargs["text_col"] + embeddings_col = kargs["embeddings_col"] + batch_size = kargs["batch_size"] + model_name = kargs["model_name"] + prefix = kargs["prefix"] + + Zipackage.install(f'{model_name}.zip') + + work_dir = os.environ.get("UPLOAD_PATH") + engine = create_embedding_engine(model_name, cache_dir=work_dir) + embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used by E5 and Harrier; Jina and SecBERT ignore it + + result = df + result[embeddings_col] = list(embeddings) ```; tbl - | evaluate hint.distribution=per_node python(typeof(*), code, kwargs, external_artifacts = bag_pack( - 'embedding_engine.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/embedding_engine.zip;impersonate', - 'harrier-v1-270m.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/harrier-v1-270m.zip;impersonate', - 'jina-v2-small.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/jina-v2-small.zip;impersonate', - 'e5-small-v2.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/e5-small-v2.zip;impersonate')) + | evaluate hint.distribution=per_node python( + typeof(*), code, kwargs, + external_artifacts=bag_pack( + 'embedding_engine.zip', strcat(artifact_root, engine_artifact, ';impersonate'), + strcat(model_name, '.zip'), strcat(artifact_root, model_name, '.zip;impersonate'))) }; // Write your query to use the function here. ~~~ @@ -88,35 +90,37 @@ Define the stored function once using the following [`.create function`](../mana .create-or-alter function with (folder = "Packages\\AI", docstring = "Embedding using local SLM") slm_embeddings_fl(tbl:(*), text_col:string, embeddings_col:string, batch_size:int=32, model_name:string='harrier-v1-270m', prefix:string='query:') { + let artifact_root = 'https://msit-onelake.dfs.fabric.microsoft.com/MY_WORKSPACE/MY_LAKEHOUSE.Lakehouse/Files/models/SLM/'; + let engine_artifact = 'embedding_engine.zip'; let kwargs = bag_pack('text_col', text_col, 'embeddings_col', embeddings_col, 'batch_size', batch_size, 'model_name', model_name, 'prefix', prefix); let code = ```if 1: - import os - from sandbox_utils import Zipackage - Zipackage.install('embedding_engine.zip') - - from embedding_factory import create_embedding_engine - - text_col = kargs["text_col"] - embeddings_col = kargs["embeddings_col"] - batch_size = kargs["batch_size"] - model_name = kargs["model_name"] - prefix = kargs["prefix"] - - Zipackage.install(f'{model_name}.zip') - - work_dir = os.environ.get("UPLOAD_PATH") # "/app/temp" - engine = create_embedding_engine(model_name, cache_dir=work_dir) - embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used only for E5 - - result = df - result[embeddings_col] = list(embeddings) + import os + from sandbox_utils import Zipackage + Zipackage.install('embedding_engine.zip') + + from embedding_factory import create_embedding_engine + + text_col = kargs["text_col"] + embeddings_col = kargs["embeddings_col"] + batch_size = kargs["batch_size"] + model_name = kargs["model_name"] + prefix = kargs["prefix"] + + Zipackage.install(f'{model_name}.zip') + + work_dir = os.environ.get("UPLOAD_PATH") + engine = create_embedding_engine(model_name, cache_dir=work_dir) + embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used by E5 and Harrier; Jina and SecBERT ignore it + + result = df + result[embeddings_col] = list(embeddings) ```; tbl - | evaluate hint.distribution=per_node python(typeof(*), code, kwargs, external_artifacts = bag_pack( - 'embedding_engine.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/embedding_engine.zip;impersonate', - 'harrier-v1-270m.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/harrier-v1-270m.zip;impersonate', - 'jina-v2-small.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/jina-v2-small.zip;impersonate', - 'e5-small-v2.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/e5-small-v2.zip;impersonate')) + | evaluate hint.distribution=per_node python( + typeof(*), code, kwargs, + external_artifacts=bag_pack( + 'embedding_engine.zip', strcat(artifact_root, engine_artifact, ';impersonate'), + strcat(model_name, '.zip'), strcat(artifact_root, model_name, '.zip;impersonate'))) } ~~~ @@ -135,35 +139,37 @@ To use a query-defined function, invoke it after the embedded function definitio ~~~kusto let slm_embeddings_fl = (tbl:(*), text_col:string, embeddings_col:string, batch_size:int=32, model_name:string='harrier-v1-270m', prefix:string='query:') { + let artifact_root = 'https://msit-onelake.dfs.fabric.microsoft.com/MY_WORKSPACE/MY_LAKEHOUSE.Lakehouse/Files/models/SLM/'; + let engine_artifact = 'embedding_engine.zip'; let kwargs = bag_pack('text_col', text_col, 'embeddings_col', embeddings_col, 'batch_size', batch_size, 'model_name', model_name, 'prefix', prefix); let code = ```if 1: - import os - from sandbox_utils import Zipackage - Zipackage.install('embedding_engine.zip') - - from embedding_factory import create_embedding_engine - - text_col = kargs["text_col"] - embeddings_col = kargs["embeddings_col"] - batch_size = kargs["batch_size"] - model_name = kargs["model_name"] - prefix = kargs["prefix"] - - Zipackage.install(f'{model_name}.zip') - - work_dir = os.environ.get("UPLOAD_PATH") # "/app/temp" - engine = create_embedding_engine(model_name, cache_dir=work_dir) - embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used only for E5 - - result = df - result[embeddings_col] = list(embeddings) + import os + from sandbox_utils import Zipackage + Zipackage.install('embedding_engine.zip') + + from embedding_factory import create_embedding_engine + + text_col = kargs["text_col"] + embeddings_col = kargs["embeddings_col"] + batch_size = kargs["batch_size"] + model_name = kargs["model_name"] + prefix = kargs["prefix"] + + Zipackage.install(f'{model_name}.zip') + + work_dir = os.environ.get("UPLOAD_PATH") + engine = create_embedding_engine(model_name, cache_dir=work_dir) + embeddings = engine.encode(df[text_col].tolist(), batch_size=batch_size, prefix=prefix) # prefix is used by E5 and Harrier; Jina and SecBERT ignore it + + result = df + result[embeddings_col] = list(embeddings) ```; tbl - | evaluate hint.distribution=per_node python(typeof(*), code, kwargs, external_artifacts = bag_pack( - 'embedding_engine.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/embedding_engine.zip;impersonate', - 'harrier-v1-270m.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/harrier-v1-270m.zip;impersonate', - 'jina-v2-small.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/jina-v2-small.zip;impersonate', - 'e5-small-v2.zip', 'https://artifactswestus.z22.web.core.windows.net/models/SLM/e5-small-v2.zip;impersonate')) + | evaluate hint.distribution=per_node python( + typeof(*), code, kwargs, + external_artifacts=bag_pack( + 'embedding_engine.zip', strcat(artifact_root, engine_artifact, ';impersonate'), + strcat(model_name, '.zip'), strcat(artifact_root, model_name, '.zip;impersonate'))) }; // // Create a sample dataset with text passages