diff --git a/ai_services/language/PLSQL/main.sql b/ai_services/language/PLSQL/main.sql new file mode 100644 index 00000000..70cd3671 --- /dev/null +++ b/ai_services/language/PLSQL/main.sql @@ -0,0 +1,75 @@ +SET SERVEROUTPUT ON; + +DECLARE + cloud_cred VARCHAR2(40) := 'OCI_AI_TEXT_CRED_C9'; + ---- + entities_details DBMS_CLOUD_OCI_AI_LANGUAGE_BATCH_DETECT_LANGUAGE_ENTITIES_DETAILS_T; + entity_document DBMS_CLOUD_OCI_AI_LANGUAGE_ENTITY_DOCUMENT_T; + entity_documents_tbl DBMS_CLOUD_OCI_AI_LANGUAGE_ENTITY_DOCUMENT_TBL; + entities_response DBMS_CLOUD_OCI_AIL_AI_SERVICE_LANGUAGE_BATCH_DETECT_LANGUAGE_ENTITIES_RESPONSE_T; + entities_result DBMS_CLOUD_OCI_AI_LANGUAGE_BATCH_DETECT_LANGUAGE_ENTITIES_RESULT_T; + result_documents_tbl DBMS_CLOUD_OCI_AI_LANGUAGE_ENTITY_DOCUMENT_RESULT_TBL; + errors_documents_tbl DBMS_CLOUD_OCI_AI_LANGUAGE_ENTITY_DOCUMENT_RESULT_TBL; + result_document DBMS_CLOUD_OCI_AI_LANGUAGE_ENTITY_DOCUMENT_T; +BEGIN + --- Init Entity Document Table + entity_documents_tbl := DBMS_CLOUD_OCI_AI_LANGUAGE_ENTITY_DOCUMENT_TBL(); + + -- Init, Create and Add 3 documents to Entity Documents Table + entity_documents_tbl.EXTEND; + entity_document := DBMS_CLOUD_OCI_AI_LANGUAGE_ENTITY_DOCUMENT_T(); + entity_document.key := 'Doc_1'; + entity_document.text := ‘Email london@gmail.com in city London for Person Jack London'; + entity_document.language_code := 'en'; + entity_documents_tbl(entity_documents_tbl.LAST) := entity_document; + + entity_documents_tbl.EXTEND; + entity_document := DBMS_CLOUD_OCI_AI_LANGUAGE_ENTITY_DOCUMENT_T(); + entity_document.key := 'Doc_2'; + entity_document.text := ‘Brands Nike, Adidas Group and Ford Corp.'; + entity_document.language_code := 'en'; + entity_documents_tbl(entity_documents_tbl.LAST) := entity_document; + + entity_documents_tbl.EXTEND; + entity_document := DBMS_CLOUD_OCI_AI_LANGUAGE_ENTITY_DOCUMENT_T(); + entity_document.key := 'Doc_3'; + entity_document.text := 'Empty text'; + entity_document.language_code := 'en'; + entity_documents_tbl(entity_documents_tbl.LAST) := entity_document; + + -- Add Entity Documents Table to Entities Details + entities_details := DBMS_CLOUD_OCI_AI_LANGUAGE_BATCH_DETECT_LANGUAGE_ENTITIES_DETAILS_T(); + entities_details.documents := entity_documents_tbl; + + -- Main function + entities_response := DBMS_CLOUD_OCI_AIL_AI_SERVICE_LANGUAGE.BATCH_DETECT_LANGUAGE_ENTITIES + ( + batch_detect_language_entities_details => entities_details, + region => 'us-ashburn-1', + credential_name => cloud_cred + ); + + entities_result := entities_response.response_body; + result_documents_tbl := entities_result.documents; + + -------------------------------------------------------------- + + dbms_output.put_line('status_code : ' || entities_response.status_code || CHR(10)); + + -- Parsing and printing results + for r in (select * from TABLE(result_documents_tbl)) + loop + dbms_output.put_line(' -- ' || r.key || ' | ' || r.language_code || ' -- '); + for m in (select * from TABLE(r.entities)) + loop + dbms_output.put_line ( + m.offset || ' | ' || + m.length || ' | ' || + m.text || ' | ' || + m.l_type || ' | ' || + m.sub_type || ' | ' || + m.score); + end loop; + dbms_output.put_line(CHR(10)); + end loop; +END; diff --git a/ai_services/language/java/src/main/java/com/company/Main.java b/ai_services/language/java/src/main/java/com/company/Main.java index da529567..295c5a4c 100644 --- a/ai_services/language/java/src/main/java/com/company/Main.java +++ b/ai_services/language/java/src/main/java/com/company/Main.java @@ -14,6 +14,21 @@ public class Main { + private static String compartmentId = ""; + private static String modelName = "ailangaugemodel"; + private static String bucket_name = "" + private static String namespace_name = "" + private static String object_name = "" + private static String projectId; + private static String modelId; + private static String nerEndpointId; + private static String txtcEndpointId; + private static String entext = "The Indy Autonomous Challenge is the worlds first head-to-head, high speed autonomous race taking place at the Indianapolis Motor Speedway"; + private static String spanishText = "Este es un texto en el idioma de mi madre, la mejor mamá del mundo."; + String languageCode = "en"; // for english + + private static AIServiceLanguageClient client; + public static void main(String[] args) { try { /* Step 1: Just reading a config file with my credentials so the application acts on my behalf */ @@ -24,12 +39,12 @@ public static void main(String[] args) { new ConfigFileAuthenticationDetailsProvider(configFile); /* Step 2: Create a service client */ - AIServiceLanguageClient client = AIServiceLanguageClient.builder().build(provider); + client = AIServiceLanguageClient.builder().build(provider); - /* Step 3: Sample of single record API */ + /*Sample of single record API */ DetectDominantLanguageDetails detectdominantLanguageDetails = DetectDominantLanguageDetails.builder() - .text("Este es un texto en el idioma de mi madre, la mejor mamá del mundo.").build(); + .text(spanishText).build(); DetectDominantLanguageRequest detectDominantLanguageRequest = DetectDominantLanguageRequest.builder() .detectDominantLanguageDetails(detectdominantLanguageDetails) @@ -40,15 +55,15 @@ public static void main(String[] args) { detectDominantLanguage(detectDominantLanguageRequest); System.out.println("Detected language: " + - response.getDetectDominantLanguageResult().getLanguages().get(0).getName()); + response.getDetectDominantLanguageResult().getLanguages().get(0).getName()); - /* Step 4: Sample using more efficient batch APIs */ + /*Sample using more efficient batch APIs */ BatchLanguageTranslationDetails batchLanguageTranslationDetails = BatchLanguageTranslationDetails.builder() .targetLanguageCode("en") .documents(new ArrayList<>(Arrays.asList(TextDocument.builder() .key("key1") - .text("El idioma español es muy facil de aprender.") + .text(spanishText) .languageCode("es").build()))).build(); BatchLanguageTranslationRequest batchLanguageTranslationRequest = BatchLanguageTranslationRequest.builder() @@ -59,10 +74,152 @@ public static void main(String[] args) { BatchLanguageTranslationResponse response1 = client.batchLanguageTranslation(batchLanguageTranslationRequest); System.out.println("Translation: " + response1.getBatchLanguageTranslationResult().getDocuments().get(0).getTranslatedText()); - - } - catch(IOException e) { + /* Custom Models */ + /* Create an object */ + Main aiServiceLanguageExample = new Main(); + + // Create AiLanguageProject + Project languageProject = aiServiceLanguageExample.createLanguageProject(); + projectId = languageProject.getId(); + System.out.println(languageProject.toString()); + + // wait till project state becomes ACTIVE + while (languageProject.getLifecycleState() == Project.LifecycleState.Creating){ + System.out.println("Waiting for project creation to complete..."); + Thread.sleep(4000); + languageProject = aiServiceLanguageExample.getLanguageProject(projectId); + } + languageProject = aiServiceLanguageExample.getLanguageProject(projectId); + System.out.println("Project status changed to" + languageProject.getLifecycleState()); + + /* Create and train Custom NER Model */ + // Create and train Custom NER AilanguageModel + Model languageModel = aiServiceLanguageExample.createLanguageModel(); + modelId = languageModel.getId(); + System.out.println(languageModel.toString()); + + // wait till model state becomes ACTIVE + while (languageModel.getLifecycleState() == Model.LifecycleState.Creating){ + System.out.println("Waiting for model training to complete..."); + Thread.sleep(60000); + languageModel = aiServiceLanguageExample.getLanguageModel(modelId); + } + languageModel = aiServiceLanguageExample.getLanguageModel(modelId); + System.out.println("Model status changed to" + languageModel.getLifecycleState()); + + System.out.println("Printing model evaluation results"); + System.out.println(languageModel.getEvaluationResults()); + + // Create AiLanguageEndpoint + Endpoint languageEndpoint = aiServiceLanguageExample.createLanguageEndpoint(); + nerEndpointId = languageEndpoint.getId(); + System.out.println(languageEndpoint.toString()); + + // wait till endpoint state becomes ACTIVE + while (languageEndpoint.getLifecycleState() == Endpoint.LifecycleState.Creating){ + System.out.println("Waiting for endpoint creation to complete..."); + Thread.sleep(60000); + languageEndpoint = aiServiceLanguageExample.getLanguageEndpoint(nerEndpointId); + } + languageEndpoint = aiServiceLanguageExample.getLanguageEndpoint(nerEndpointId); + System.out.println("Endpoint status changed to" + languageEndpoint.getLifecycleState()); + + // Inferencing on Custom Named Entity recognition model + String customDetectLanguageEntitiesResponse = aiServiceLanguageExample.getBatchDetectLanguageEntities(text); + System.out.println(customDetectLanguageEntitiesResponse.toString()); + + /* Create and train Custom Text classification Model */ + + // Create and train Custom Text classification AilanguageModel + languageModel = aiServiceLanguageExample.createLanguageModel(); + modelId = languageModel.getId(); + System.out.println(languageModel.toString()); + + // Create AiLanguageEndpoint + languageEndpoint = aiServiceLanguageExample.createLanguageEndpoint(); + txtcEndpointId = languageEndpoint.getId(); + System.out.println(languageEndpoint.toString()); + + // Inferencing on Custom Text classification model + String customDetectLanguageTextClassificationResponse = aiServiceLanguageExample.getBatchDetectLanguageTextClassification(text); + System.out.println(customDetectLanguageTextClassificationResponse.toString()); + + client.close(); + + } catch (IOException e) { e.printStackTrace(); } } + + // Create AiLanguageProject + private Project createLanguageProject() { + CreateProjectDetails projectDetails = CreateProjectDetails.builder().compartmentId(compartmentId).build(); + CreateProjectRequest request = CreateProjectRequest.builder().createProjectDetails(projectDetails).build(); + CreateProjectResponse response = client.createProject(request); + return response.getProject(); + } + + // Get AiLanguageProject + private Project getLanguageProject(String projectOcid) { + GetProjectRequest request = GetProjectRequest.builder().projectId(projectOcid).build(); + GetProjectResponse response = client.getProject(request); + return response.getProject(); + } + + // Create AiLanguageModel + private Model createLanguageModel() { + ModelDetails modeldtls = NamedEntityRecognitionModelDetails.builder().languageCode("en").build(); + java.util.List trainingDataobjects = Arrays.asList(object_name); + LocationDetails locationDetails = ObjectListDataset.builder().bucketName(bucket_name).namespaceName(namespace_name).objectNames(trainingDataobjects).build(); + DatasetDetails trainingDataset = ObjectStorageDataset.builder().locationDetails(locationDetails).build(); + + CreateModelDetails modelDetails = CreateModelDetails.builder() + .compartmentId(compartmentId).displayName(modelName).projectId(projectId) + .modelDetails(modeldtls).trainingDataset(trainingDataset).build(); + CreateModelRequest request = CreateModelRequest.builder().createModelDetails(modelDetails).build(); + CreateModelResponse response = client.createModel(request); + return response.getModel(); + } + + // Get AiLanguageModel + private Model getLanguageModel(String modelOcid) { + GetModelRequest request = GetModelRequest.builder().modelId(modelOcid).build(); + GetModelResponse response = client.getModel(request); + return response.getModel(); + } + + // Create AiLanguageEndpoint + private Endpoint createLanguageEndpoint() { + CreateEndpointDetails endpointDetails = CreateEndpointDetails.builder().compartmentId(compartmentId).modelId(modelId).inferenceUnits(1).build(); + CreateEndpointRequest request = CreateEndpointRequest.builder().createEndpointDetails(endpointDetails).build(); + CreateEndpointResponse response = client.createEndpoint(request); + return response.getEndpoint(); + } + + // Get AiLanguageEndpoint + private Endpoint getLanguageEndpoint(String endpointOcid) { + GetEndpointRequest request = GetEndpointRequest.builder().endpointId(endpointOcid).build(); + GetEndpointResponse response = client.getEndpoint(request); + return response.getEndpoint(); + } + + // Custom Named Entity Recognition + private String getBatchDetectLanguageEntities(String text) { + TextDocument textDocument = TextDocument.builder().text(text).languageCode("en").key("key1").build(); + java.util.List documents = Arrays.asList(textDocument); + BatchDetectLanguageEntitiesDetails details = BatchDetectLanguageEntitiesDetails.builder().endpointId(nerEndpointId).documents(documents).build(); + BatchDetectLanguageEntitiesRequest request = BatchDetectLanguageEntitiesRequest.builder().batchDetectLanguageEntitiesDetails(details).build(); + BatchDetectLanguageEntitiesResponse response = client.batchDetectLanguageEntities(request); + return response.toString(); + } + + // Custom Text Classification + private String getBatchDetectLanguageTextClassification(String text) { + TextDocument textDocument = TextDocument.builder().text(text).languageCode("en").key("key1").build(); + java.util.List documents = Arrays.asList(textDocument); + BatchDetectLanguageTextClassificationDetails details = BatchDetectLanguageTextClassificationDetails.builder().endpointId(txtcEndpointId).documents(documents).build(); + BatchDetectLanguageTextClassificationRequest request = BatchDetectLanguageTextClassificationRequest.builder().batchDetectLanguageTextClassificationDetails(details).build(); + BatchDetectLanguageTextClassificationResponse response = client.batchDetectLanguageTextClassification(request); + return response.toString(); + } } diff --git a/ai_services/language/python/languageCustomNERSample.py b/ai_services/language/python/languageCustomNERSample.py new file mode 100644 index 00000000..8695f97c --- /dev/null +++ b/ai_services/language/python/languageCustomNERSample.py @@ -0,0 +1,94 @@ +languageCustomTXTCSample.pyimport time +import oci +print(f'OCI Client SDK version: {oci.__version__}') +config = oci.config.from_file() + +compartment_id = #TODO Specify your compartmentId here +bucket_name = #TODO Specify name of your training data bucket here +namespace_name = #TODO Specify the namespace here +object_names = ["CNER_offer_trainset_01_labels.jsonl"] + +project_name = #TODO specify project name here +model_name = #TODO specify the model name here +endpoint_name = #TODO specify endpoint name + +ai_client = oci.ai_language.AIServiceLanguageClient(config) + +#create project +project_details = oci.ai_language.models.CreateProjectDetails(compartment_id=compartment_id,display_name=project_name) +print(f"Creating project with details:{project_details}") +project = ai_client.create_project(project_details) +print(f"create_project returned: {project.data}") + +#wait till project state becomes ACTIVE +project_id = project.data.id +project_status = project.data.lifecycle_state +while (project.data.lifecycle_state == "CREATING"): + print('Waiting for project creation to complete...') + time.sleep(1*60) #sleep for 1 minute + project = ai_client.get_project(project_id) + +project = ai_client.get_project(project_id) +project_status = project.data.lifecycle_state +print(f"Project status changed from CREATING to {project_status}") + +#creating model +location_details = oci.ai_language.models.ObjectListDataset( + location_type="OBJECT_LIST", + namespace_name=namespace_name, + bucket_name=bucket_name, + object_names=object_names +) + +model_details = oci.ai_language.models.CreateModelDetails( + project_id = project.data.id, + model_details = oci.ai_language.models.ModelDetails(model_type="NAMED_ENTITY_RECOGNITION"), + display_name = model_name, + compartment_id = compartment_id, + training_dataset = oci.ai_language.models.ObjectStorageDataset(dataset_type="OBJECT_STORAGE", location_details=location_details) +) + +print(f"creating model with details:{model_details}") +model_response = ai_client.create_model(model_details) +print(f"create_model returned: {model_response.data}") + +model_details = ai_client.get_model(model_response.data.id) + +#wait till model state becomes ACTIVE +while (model_details.data.lifecycle_state == "CREATING"): + print('Waiting for model creation and training to complete...') + time.sleep(1*60) #sleep for 1 minute + model_details = ai_client.get_model(model_response.data.id) + +print(f"Model status changed from CREATING to {model_details.data.lifecycle_state}") + +print("Printing model evaluation results") +print(model_details.data.evaluation_results) + +print("Creating an end point") +endpoint_details = oci.ai_language.models.CreateEndpointDetails( + compartment_id = compartment_id, + model_id = model_details.data.id, + inference_units = 1, + display_name = endpoint_name +) + +endpoint_response = ai_client.create_endpoint(endpoint_details) +print(f"create_endpoint call returned{endpoint_response.data}") +end_point_details = ai_client.get_endpoint(endpoint_response.data.id) + +#wait till endpoint state becomes ACTIVE +while (end_point_details.data.lifecycle_state == "CREATING"): + print('Waiting for endpoint creation to complete...') + time.sleep(1*60) #sleep for 5 minutes + end_point_details = ai_client.get_endpoint(end_point_details.data.id) + +print(f"End point status changed from CREATING to {end_point_details.data.lifecycle_state}") + +text_to_analyze = "\n\nDear Bryan Hernandez,\n \nGilmore Kennedy and Lloyd is delighted to offer you the position of Chief Strategy Officer with an anticipated start date of 06/16/17, contingent upon background check, drug screening and work permit verification. \n \nYou will report directly to Jeffrey Zamora at Unit 6709 Box 6713,DPO AP 11187. Working hours are decided based on your assigned business unit. \n \nThe starting salary for this position is $216053 per annum. Payment is on a monthly basis by direct deposit done on the last working day of the moth. \n \nGilmore Kennedy and Lloyd offers a comprehensive benefits program, which includes medical insurance, 401(k), paid time off and gym facilities at work location. \n \nYour employment with Gilmore Kennedy and Lloyd will be on an at-will basis, which means you and the company are free to terminate employment at any time, with or without cause or advance notice. This letter is not a contract indicating employment terms or duration.\n \nPlease confirm your acceptance of this offer by signing and returning this letter before 7 days from 06/16/17. \n \nSincerely,\n \nCarlos Banks\n(Country Leader, Human Resources)" + +print(f"Analyzing the text: {text_to_analyze}") +ner_text_for_testing = oci.ai_language.models.BatchDetectLanguageEntitiesDetails(endpoint_id = end_point_details.data.id, documents = [oci.ai_language.models.TextDocument(key = "1", text = text_to_analyze)]) +ner_inference_result = ai_client.batch_detect_language_entities(ner_text_for_testing) +print("inference result for custom NER:") +print(ner_inference_result.data) diff --git a/ai_services/language/python/languageCustomTXTCSample.py b/ai_services/language/python/languageCustomTXTCSample.py new file mode 100644 index 00000000..de980bdb --- /dev/null +++ b/ai_services/language/python/languageCustomTXTCSample.py @@ -0,0 +1,90 @@ +import time +import oci +print(f'OCI Client SDK version: {oci.__version__}') +config = oci.config.from_file() + +compartment_id = #TODO Specify your compartmentId here +bucket_name = #TODO Specify name of your training data bucket here +namespace_name = #TODO Specify the namespace here +object_names = ["TicketData_train.csv"] + +project_name = #"custom-TXTC-project" +model_name = #"custom-TXTC-model" +endpoint_name = #"custom_TXTC_endpoint" + +ai_client = oci.ai_language.AIServiceLanguageClient(config) + +#create project +project_details = oci.ai_language.models.CreateProjectDetails(compartment_id=compartment_id,display_name=project_name) +print(f"Creating project with details:{project_details}") +project = ai_client.create_project(project_details) +print(f"create_project returned: {project.data}") + +#wait till project state becomes ACTIVE +project_id = project.data.id +project_status = project.data.lifecycle_state +while (project.data.lifecycle_state == "CREATING"): + print('Waiting for project creation to complete...') + time.sleep(1*60) #sleep for 1 minute + project = ai_client.get_project(project_id) + +project = ai_client.get_project(project_id) +project_status = project.data.lifecycle_state +print(f"Project status changed from CREATING to {project_status}") + +#creating model +location_details = oci.ai_language.models.ObjectListDataset(location_type="OBJECT_LIST", namespace_name=namespace_name, bucket_name=bucket_name, object_names=object_names) +# For Text classification, multi-class and multi-labe classification types are supported +classification_mode = oci.ai_language.models.ClassificationType(classification_mode="MULTI_CLASS") +model_details = oci.ai_language.models.CreateModelDetails( + project_id = project.data.id, + model_details = oci.ai_language.models.TextClassificationModelDetails(classification_mode=classification_mode,model_type="TEXT_CLASSIFICATION", language_code="en"), + display_name = model_name, + compartment_id = compartment_id, + training_dataset = oci.ai_language.models.ObjectStorageDataset(dataset_type="OBJECT_STORAGE", location_details=location_details) +) + +print(f"creating model with details:{model_details}") +model_response = ai_client.create_model(model_details) +print(f"create_model returned: {model_response.data}") + +model_details = ai_client.get_model(model_response.data.id) + +#wait till model state becomes ACTIVE +while (model_details.data.lifecycle_state == "CREATING"): + print('Waiting for model creation and training to complete...') + time.sleep(1*60) #sleep for 1 minute + model_details = ai_client.get_model(model_response.data.id) + +print(f"Model status changed from CREATING to {model_details.data.lifecycle_state}") + +print("Printing model evaluation results") +print(model_details.data.evaluation_results) + +print("Creating an end point") +endpoint_details = oci.ai_language.models.CreateEndpointDetails( + compartment_id = compartment_id, + model_id = model_details.data.id, + inference_units = 1, + display_name = endpoint_name +) + +endpoint_response = ai_client.create_endpoint(endpoint_details) +print(f"create_endpoint call returned{endpoint_response.data}") +end_point_details = ai_client.get_endpoint(endpoint_response.data.id) + +#wait till endpoint state becomes ACTIVE +while (end_point_details.data.lifecycle_state == "CREATING"): + print('Waiting for endpoint creation to complete...') + time.sleep(1*60) #sleep for 5 minutes + end_point_details = ai_client.get_endpoint(end_point_details.data.id) + +print(f"End point status changed from CREATING to {end_point_details.data.lifecycle_state}") + +text_to_analyze = "I am unable to use printer, seeing printer error." + +print(f"Analyzing the text: {text_to_analyze}") +txtc_text_for_testing = oci.ai_language.models.BatchDetectLanguageEntitiesDetails(endpoint_id = end_point_details.data.id, documents = [oci.ai_language.models.TextDocument(key = "1", text = text_to_analyze)]) +txtc_inference_result = ai_client.batch_detect_language_entities(txtc_text_for_testing) +print("inference result for custom TXTC:") +print(txtc_inference_result.data) diff --git a/ai_services/language/python/languagetranslation.py b/ai_services/language/python/languagetranslation.py new file mode 100644 index 00000000..97f61d8e --- /dev/null +++ b/ai_services/language/python/languagetranslation.py @@ -0,0 +1,20 @@ +import oci + +ai_client = oci.ai_language.AIServiceLanguageClient(oci.config.from_file()) + +key1 = "doc1" +key2 = "doc2" +text1 = "The Indy Autonomous Challenge is the worlds first head-to-head, high speed autonomous race taking place at the Indianapolis Motor Speedway" +text2 = "OCI will be the cloud engine for the artificial intelligence models that drive the MIT Driverless cars." +target_language = "de" #TODO specify the target language +compartment_id = #TODO Provide your compartmentId here + +doc1 = oci.ai_language.models.TextDocument(key=key1, text=text1, language_code="en") +doc2 = oci.ai_language.models.TextDocument(key=key2, text=text2, language_code="en") +documents = [doc1, doc2] + + + +batch_language_translation_details = oci.ai_language.models.BatchLanguageTranslationDetails(documents=documents, compartment_id=compartment_id, target_language_code=target_language) +output = ai_client.batch_language_translation (batch_language_translation_details) +print(output.data)