init

1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 1024,
+  "pooling_mode_cls_token": true,
+  "pooling_mode_mean_tokens": false,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

README.md

@@ -1,3 +1,87 @@
----
-license: mit
----
+# ember-v1
+
+<p align="center">
+<img src="https://console.llmrails.com/assets/img/logo-black.svg" width="150px">
+</p>
+
+This model is trained on a large-scale corpus of relevance text pairs, covering a wide range of domains like financial, scientific, medical, legal  and others. While training we used some technics from Retromae and SetFit papers.
+
+We are also providing it on our own platform as API as a service, feel free to signup: [LLMRails](https://llmrails.com/?ref=ember-v1).
+
+### Plans
+- Paper will be published soon
+- v2 is on it's way with 4k maximum sequence length
+
+## Usage
+Use with API request:
+```bash
+curl --location 'https://api.llmrails.com/v1/embeddings' \
+--header 'X-API-KEY: {token}' \
+--header 'Content-Type: application/json' \
+--data '{
+   "input": ["This is an example sentence"],
+   "model":"embedding-english-v1" # equals to ember-v1
+}'
+```
+API docs: https://docs.llmrails.com/embedding/embed-text
+Langchain plugin: https://python.langchain.com/docs/integrations/text_embedding/llm_rails
+
+Use with transformers:
+```python
+import torch.nn.functional as F
+from torch import Tensor
+from transformers import AutoTokenizer, AutoModel
+
+def average_pool(last_hidden_states: Tensor,
+                 attention_mask: Tensor) -> Tensor:
+    last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)
+    return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
+
+input_texts = [
+    "This is an example sentence",
+    "Each sentence is converted"
+]
+
+tokenizer = AutoTokenizer.from_pretrained("llmrails/ember-v1")
+model = AutoModel.from_pretrained("llmrails/ember-v1")
+
+# Tokenize the input texts
+batch_dict = tokenizer(input_texts, max_length=512, padding=True, truncation=True, return_tensors='pt')
+
+outputs = model(**batch_dict)
+embeddings = average_pool(outputs.last_hidden_state, batch_dict['attention_mask'])
+
+# (Optionally) normalize embeddings
+embeddings = F.normalize(embeddings, p=2, dim=1)
+scores = (embeddings[:1] @ embeddings[1:].T) * 100
+print(scores.tolist())
+```
+
+Use with sentence-transformers:
+```python
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.util import cos_sim
+
+sentences = [
+    "This is an example sentence",
+    "Each sentence is converted"
+]
+
+model = SentenceTransformer('llmrails/ember-v1')
+embeddings = model.encode(sentences)
+print(cos_sim(embeddings[0], embeddings[1]))
+```
+
+## Massive Text Embedding Benchmark (MTEB) Evaluation
+Our model achieve state-of-the-art performance on [MTEB leaderboard](https://huggingface.co/spaces/mteb/leaderboard)
+
+|                               Model Name                                | Dimension | Sequence Length | Average (56) | 
+|:-----------------------------------------------------------------------:|:---------:|:---:|:------------:|
+| [bge-large-en-v1.5](https://huggingface.co/BAAI/bge-large-en-v1.5) |   1024    |       512       |    64.23     |  
+| [bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5) |   768    |       512       |    63.55     |
+| [ember-v1](https://huggingface.co/llmrails/emmbedding-en-v1) |   1024    | 512 |    **63.54**     |  
+| [text-embedding-ada-002](https://platform.openai.com/docs/guides/embeddings/types-of-embedding-models) |   1536    |      8191       |    60.99     |
+
+### Limitation
+
+This model exclusively caters to English texts, and any lengthy texts will be truncated to a maximum of 512 tokens.

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "/root/.cache/torch/sentence_transformers/llmrails_luna-v1/",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.33.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.2.2",
+    "transformers": "4.28.1",
+    "pytorch": "1.13.0+cu117"
+  }
+}

modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  }
+]

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63cd396b456f848b7e643f3d6a703a01c7b08337519ee1bf5accbc12c8ea1998
+size 1340699369

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": true
+}

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}