OpenSearch の基本的な検索機能¶

cloudformation_stack_name = "search-lab-jp"
opensearch_cluster_endpoint = search_cloudformation_output(cloudformation_stack_name, "OpenSearchDomainEndpoint")

credentials = boto3.Session().get_credentials()
service_code = "es"
auth = AWSV4SignerAuth(credentials=credentials, region=default_region, service=service_code)
opensearch_client = OpenSearch(
    hosts=[{"host": opensearch_cluster_endpoint, "port": 443}],
    http_compress=True, 
    http_auth=auth,
    use_ssl=True,
    verify_certs=True,
    connection_class = RequestsHttpConnection
)
opensearch_client.info()

index_name = "movies"

payload = {
  "settings": {
    "index": {
      "number_of_shards": 1,
      "number_of_replicas": 0
    },
    "analysis": {
      "normalizer": {
        "lowercase_normalizer": {
          "type": "custom",
          "filter": ["lowercase"]
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "id":  { "type": "keyword" },
      "directors":  { "type": "text" },
      "release_date": {"type": "date"},
      "rating": {"type": "scaled_float", "scaling_factor": 10},
      "genres":  {
        "type": "keyword",
        "normalizer": "lowercase_normalizer"
      },
      "image_url":  { "type": "keyword" },
      "plot":  {
        "type": "text",
        "analyzer": "english"
      },
      "title":  { "type": "text" },
      "rank": { "type": "integer" },
      "running_time_secs": { "type": "integer" },
      "actors":  { "type": "text" },
      "year": {"type": "short"},
      "type": { "type": "keyword" } 
    }
  }
}

try:
    # 既に同名のインデックスが存在する場合、いったん削除を行う
    print("# delete index")
    response = opensearch_client.indices.delete(index=index_name)
    print(json.dumps(response, indent=2))
except Exception as e:
    print(e)

# インデックスの作成を行う
print("# create index")
response = opensearch_client.indices.create(index=index_name, body=payload)
print(json.dumps(response, indent=2))

%%time
index_name = "movies"
file_path = f"{dataset_dir}/sample-movies.jsonl"

response = wr.opensearch.index_json(
    client=opensearch_client,
    path=file_path,
    use_threads=True, #クライアントの CPU 数に応じて自動で書き込みを並列化
    id_keys=["id"],
    index=index_name,
    bulk_size=200, # 200 件ずつ書き込み,
    refresh=False # 書き込み処理実行中の refresh (セグメントの書き出し) を無効化
)

response["success"] == len(open(file_path).readlines())

payload = {
  "text": "OpenSearch is a distributed search and analytics engine based on Apache Lucene.",
  "analyzer": "standard"
}

response = opensearch_client.indices.analyze(
  body=payload
)
df_standard = pd.json_normalize(response["tokens"])
print(json.dumps(response, indent=2))

payload = {
  "text": "OpenSearch is a distributed search and analytics engine based on Apache Lucene.",
  "analyzer": "english"
}

response = opensearch_client.indices.analyze(
  body=payload
)

df_english = pd.json_normalize(response["tokens"])
print(json.dumps(response, indent=2))

pd.merge(df_standard, df_english, on=["start_offset","end_offset", "position", "type"], how="left").rename(columns={"token_x": "token_standard", "token_y": "token_english"}).reindex(["start_offset","end_offset","position","type","token_standard","token_english"],axis=1).fillna("")

index_name = "movies"

payload = {
  "query": {
    "match": {
      "title": "avengers"
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

print(json.dumps(response, indent=2))

pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "query": {
    "match": {
      "title": "avenger"
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "match": {
      "plot": "superhero"
    }
  },
  "highlight": {
    "fields": {
      "plot": {}
    }
  } 
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "query": {
    "match": {
      "title": {
        "query": "avenger avengers"
      }
    }
  },
  "highlight": {
    "fields": {
      "plot": {}
    }
  } 
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "query": {
    "match": {
      "plot": {
        "query": "superhero transform academy",
        "operator": "and"
      }
    }
  },
  "highlight": {
    "fields": {
      "plot": {}
    }
  } 
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "query": {
    "match": {
      "plot": {
        "query": "superhero transform academy",
        "minimum_should_match": "2"
      }
    }
  },
  "highlight": {
    "fields": {
      "plot": {}
    }
  } 
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

pd.json_normalize(response["hits"]["hits"])
#print(json.dumps(response, indent=2))

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "multi_match": {
      "query": "wind",
      "fields": ["title", "plot"]
    }
  },
  "highlight": {
    "fields": {
      "title": {},
      "plot": {}
    }
  } 
}

response = opensearch_client.search(
    index=index_name,
    body=payload,
    filter_path="hits.hits"
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "query": {
    "match": {
      "title": {
        "query": "iron man",
        "operator": "and"
      }
    }
  },
  "size": 5
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "query": {
    "match_phrase": {
      "title": {
        "query": "iron man",
      }
    }
  },
  "size": 5
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "query": {
    "match_phrase": {
      "title": {
        "query": "man iron 2",
        "slop": 2
      }
    }
  },
  "size": 5
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "query": {
    "match_phrase": {
      "title": {
        "query": "man 2 iron",
        "slop": 3
      }
    }
  },
  "size": 5
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "term": {
      "genres": {
        "value": "Comedy"
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "term": {
      "genres": {
        "value": "cOmEdY"
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

print(json.dumps(response, indent=2))

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "terms": {
      "genres": ["comedy", "drama"]
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "terms_set": {
      "genres": {
        "terms": ["comedy", "drama", "sci-fi", "family"],
        "minimum_should_match_script": {
          "source": "params.num_terms"
        }
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

lookup_index_name = "movie-users"

payload = {
  "settings": {
    "index": {
      "number_of_shards": 1,
      "number_of_replicas": 0
    }
  },
  "mappings": {
    "properties": {
      "userid":  { "type": "keyword" },
      "favorite-genres":  { "type": "keyword" }
    }
  }
}

try:
    # 既に同名のインデックスが存在する場合、いったん削除を行う
    print("# delete index")
    response = opensearch_client.indices.delete(index=lookup_index_name)
    print(json.dumps(response, indent=2))
except Exception as e:
    print(e)

# インデックスの作成を行う
print("# create index")
response = opensearch_client.indices.create(index=lookup_index_name, body=payload)
print(json.dumps(response, indent=2))

lookup_index_name = "movie-users"

payload = {
  "userid": "00000001",
  "favorite-genres": ["comedy", "drama"]
}

response = opensearch_client.index(
    index = lookup_index_name,
    body = payload,
    id = "00000001",
    refresh = True
)

print(json.dumps(response, indent=2))

index_name = "movies"
lookup_index_name = "movie-users"

payload = {
  "size": 3,
  "query": {
    "terms": {
      "genres": {
        "index": lookup_index_name,
        "id": "00000001",
        "path": "favorite-genres"
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "prefix": {
      "genres": "com"
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 1,
  "query": {
    "regexp": {
      "genres": "[a-z]{5}"
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 1,
  "query": {
    "wildcard": {
      "genres": "c*y"
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 1,
  "query": {
    "fuzzy": {
      "genres": {
        "value": "cmdy",
        "fuzziness": 2
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 1,
  "query": {
    "fuzzy": {
      "genres": {
        "value": "ocmedy",
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 1,
  "query": {
    "fuzzy": {
      "genres": {
        "value": "ocemdy",
        "fuzziness": 1
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 1,
  "query": {
    "fuzzy": {
      "genres": {
        "value": "ocemdy",
        "fuzziness": 2
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 1,
  "query": {
    "fuzzy": {
      "genres": {
        "value": "ocemdy",
        "fuzziness": 2,
        "transpositions": False
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 1,
  "query": {
    "fuzzy": {
      "genres": {
        "value": "mocedy",
        "fuzziness": 1,
        "transpositions": True
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 1,
  "query": {
    "fuzzy": {
      "genres": {
        "value": "mocedy",
        "fuzziness": 2,
        "transpositions": False
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 1,
  "query": {
    "fuzzy": {
      "genres": {
        "value": "nocedy",
        "fuzziness": 2,
        "transpositions": False
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "match": {
      "plot": "superhero"
    }
  },
  "highlight": {
    "fields": {
      "plot": {}
    }
  },
  "explain": True
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

# print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "multi_match": {
      "query": "wind",
      "fields": ["title^3", "plot"]
    }
  },
  "highlight": {
    "fields": {
      "title": {},
      "plot": {}
    }
  },
  "explain": True
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "script_score": {
      "query": {
        "match": { 
            "plot": "superhero" 
        }
      },
      "script": {
        "source": "_score / doc['rank'].value"
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "script_score": {
      "query": {
        "match": { 
            "plot": "superhero" 
        }
      },
      "script": {
        "source": "sigmoid(_score, 2, 1)"
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 3,
  "query": {
    "boosting": {
      "positive": {
        "match": {
          "plot": "superhero"
        }
      },
      "negative": {
        "match": {
          "plot": "private"
        }
      },
      "negative_boost": 0.9
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 5,
  "query": {
    "match_all": {}
  },
  "sort": [
    {
      "year": {
        "order": "desc"
      }
    }
  ]
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 5,
  "query": {
    "match_all": {}
  },
  "sort": [
    {
      "year": {
        "order": "desc"
      }
    },
    {
      "id": {
        "order": "asc"
      }
    }
  ]
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "query": {
    "range": {
      "year": {
        "gte": 1920,
        "lte": 1923
      }
    }
  }
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])

index_name = "movies"

payload = {
  "size": 5,
  "query": {
    "bool": {
      "must": [
        {
          "match": {
            "plot": "school"
          }
        },
        {
          "term": {
            "genres": {
              "value": "Music"
            }
          }
        }
      ],
      "should": [
        {
          "term": {
            "genres": {
              "value": "Romance"
            }
          }
        },
        {
          "term": {
            "genres": {
              "value": "Comedy"
            }
          }
        }
      ],
      "minimum_should_match": 1,
      "filter": [
        {
          "range": {
            "year": {
              "gte": 2000,
              "lte": 2009
            }
          }
        }
      ],
      "must_not": [
        {
          "terms": {
            "genres": ["Thriller", "Horror"]
          }
        }
      ]
    }
  },
  "highlight": {
    "fields": {
      "plot": {}
    }
  },
  "sort": [
    {
       "rating": {
          "order": "desc"
      }
    }
  ]
}

response = opensearch_client.search(
    index=index_name,
    body=payload
)

#print(json.dumps(response, indent=2))
pd.json_normalize(response["hits"]["hits"])