From 324b80fec9e4d1c470b7f6b49dbaead3d86878b9 Mon Sep 17 00:00:00 2001 From: Frank Mai Date: Tue, 7 Jul 2020 13:59:43 +0800 Subject: [PATCH] refactor(mqtt): adjust mqtt adaptor - reuse mqtt util --- .../mqtt/api/v1alpha1/mqttdevice_types.go | 225 ++++--- .../api/v1alpha1/zz_generated.deepcopy.go | 201 +++--- adaptors/mqtt/deploy/e2e/all_in_one.yaml | 621 ++++++++++++++---- .../dl_attributed_message_bedroom_light.yaml | 76 +++ .../e2e/dl_attributed_topic_kitchen_door.yaml | 75 +++ .../dl_attributed_topic_kitchen_light.yaml | 137 ++++ .../dl_attributed_topic_kitchen_monitor.yaml | 50 ++ .../dl_attributed_topic_livingroom_light.yaml | 82 +++ adaptors/mqtt/deploy/e2e/roomlightcase1.yaml | 89 --- .../devices.edge.cattle.io_mqttdevices.yaml | 612 +++++++++++++---- .../deploy/manifests/crd/kustomization.yaml | 8 +- adaptors/mqtt/hack/lib/constant.sh | 0 adaptors/mqtt/hack/make-rules/adaptor.sh | 22 + adaptors/mqtt/pkg/adaptor/service.go | 96 +-- adaptors/mqtt/pkg/physical/converter.go | 180 ----- adaptors/mqtt/pkg/physical/converter_test.go | 274 -------- adaptors/mqtt/pkg/physical/device.go | 450 ++++++------- adaptors/mqtt/pkg/physical/handler.go | 5 +- adaptors/mqtt/pkg/physical/parameters.go | 28 - adaptors/mqtt/pkg/physical/validation.go | 40 ++ adaptors/mqtt/pkg/physical/validation_test.go | 63 ++ adaptors/mqtt/test/e2e/.gitkeep | 0 adaptors/mqtt/test/integration/.gitkeep | 0 .../integration/adaptor/connection_test.go | 130 ---- .../test/integration/adaptor/suite_test.go | 37 -- adaptors/mqtt/test/quickstart.md | 97 --- .../testdevice/roomlight/cmd/roomlight.go | 147 ----- .../testdata/testdevice/roomlight/main.go | 16 - go.mod | 2 +- go.sum | 7 +- hack/make-rules/template-adaptor.sh | 6 +- pkg/mqtt/client.go | 7 +- .../deploy/manifests/crd/kustomization.yaml | 2 +- vendor/github.com/tidwall/pretty/README.md | 2 +- vendor/github.com/tidwall/pretty/pretty.go | 34 +- vendor/github.com/tidwall/sjson/go.mod | 8 - vendor/github.com/tidwall/sjson/go.sum | 7 - vendor/github.com/tidwall/sjson/sjson.go | 193 +----- vendor/github.com/tidwall/sjson/sjson_gae.go | 196 ++++++ vendor/github.com/tidwall/sjson/sjson_ngae.go | 191 ++++++ vendor/modules.txt | 4 +- 41 files changed, 2418 insertions(+), 2002 deletions(-) create mode 100644 adaptors/mqtt/deploy/e2e/dl_attributed_message_bedroom_light.yaml create mode 100644 adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_door.yaml create mode 100644 adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_light.yaml create mode 100644 adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_monitor.yaml create mode 100644 adaptors/mqtt/deploy/e2e/dl_attributed_topic_livingroom_light.yaml delete mode 100644 adaptors/mqtt/deploy/e2e/roomlightcase1.yaml mode change 100755 => 100644 adaptors/mqtt/hack/lib/constant.sh delete mode 100644 adaptors/mqtt/pkg/physical/converter.go delete mode 100644 adaptors/mqtt/pkg/physical/converter_test.go delete mode 100644 adaptors/mqtt/pkg/physical/parameters.go create mode 100644 adaptors/mqtt/pkg/physical/validation.go create mode 100644 adaptors/mqtt/pkg/physical/validation_test.go create mode 100644 adaptors/mqtt/test/e2e/.gitkeep create mode 100644 adaptors/mqtt/test/integration/.gitkeep delete mode 100644 adaptors/mqtt/test/integration/adaptor/connection_test.go delete mode 100644 adaptors/mqtt/test/integration/adaptor/suite_test.go delete mode 100644 adaptors/mqtt/test/quickstart.md delete mode 100644 adaptors/mqtt/test/testdata/testdevice/roomlight/cmd/roomlight.go delete mode 100644 adaptors/mqtt/test/testdata/testdevice/roomlight/main.go delete mode 100644 vendor/github.com/tidwall/sjson/go.mod delete mode 100644 vendor/github.com/tidwall/sjson/go.sum create mode 100644 vendor/github.com/tidwall/sjson/sjson_gae.go create mode 100644 vendor/github.com/tidwall/sjson/sjson_ngae.go diff --git a/adaptors/mqtt/api/v1alpha1/mqttdevice_types.go b/adaptors/mqtt/api/v1alpha1/mqttdevice_types.go index 43bf8a47..42cbc8a0 100644 --- a/adaptors/mqtt/api/v1alpha1/mqttdevice_types.go +++ b/adaptors/mqtt/api/v1alpha1/mqttdevice_types.go @@ -1,159 +1,184 @@ package v1alpha1 import ( - "fmt" - "strconv" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" + + mqttapi "github.com/rancher/octopus/pkg/mqtt/api" ) +// MQTTDevicePropertyType defines the type of the property value. +// +kubebuilder:validation:Enum=int;string;float;boolean;object;array +type MQTTDevicePropertyType string + const ( - // Device Property value type - ValueTypeInt ValueType = "int" - ValueTypeString ValueType = "string" - ValueTypeFloat ValueType = "float" - ValueTypeBoolean ValueType = "boolean" - ValueTypeArray ValueType = "array" - ValueTypeObject ValueType = "object" - - // Subscribed topic payload type - PayloadTypeJSON PayloadType = "json" + MQTTDevicePropertyTypeInt MQTTDevicePropertyType = "int" + MQTTDevicePropertyTypeString MQTTDevicePropertyType = "string" + MQTTDevicePropertyTypeFloat MQTTDevicePropertyType = "float" + MQTTDevicePropertyTypeBoolean MQTTDevicePropertyType = "boolean" + MQTTDevicePropertyTypeObject MQTTDevicePropertyType = "object" + MQTTDevicePropertyTypeArray MQTTDevicePropertyType = "array" ) -// Defines the type of the property value. -// +kubebuilder:validation:Enum=int;string;float;boolean;array;object -type ValueType string +// MQTTDevicePattern defines the pattern that published/subscribed the message. +// AttributedMessage: Compress properties into one message, one topic has its all property values. +// AttributedTopic: Flatten properties to topic, each topic has its own property value. +// +kubebuilder:validation:Enum=AttributedMessage;AttributedTopic +type MQTTDevicePattern string -// The payload type type. -// +kubebuilder:validation:Enum=json -type PayloadType string +const ( + MQTTDevicePatternAttributedMessage MQTTDevicePattern = "AttributedMessage" + MQTTDevicePatternAttributeTopic MQTTDevicePattern = "AttributedTopic" +) -// The qos type. -// +kubebuilder:validation:Enum=0;1;2 -type QosType int +// MQTTDeviceSchema defines the pattern schema. +type MQTTDeviceSchema struct { + // Specifies the type of schema. + // +optional + Type string `json:"type,omitempty"` -type MqttConfig struct { - Broker string `json:"broker"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` + // Specifies the reference for schema. + // +optional + Reference string `json:"reference,omitempty"` } -type SubInfo struct { - Topic string `json:"topic"` - PayloadType PayloadType `json:"payloadType"` - Qos QosType `json:"qos"` -} +// MQTTDeviceProtocol is the Schema for configuring the protocol of MQTTDevice. +type MQTTDeviceProtocol struct { + mqttapi.MQTTOptions `json:",inline"` -type PubInfo struct { - Topic string `json:"topic"` - Qos QosType `json:"qos"` -} + // Specifies the pattern of MQTTDevice protocol. + // +kubebuilder:validation:Required + Pattern MQTTDevicePattern `json:"pattern"` -type ValueFloat struct { - F float64 `json:"-"` + // Specifies the schema of the pattern. + // +optional + Schema *MQTTDeviceSchema `json:"schema,omitempty"` } -func (v *ValueFloat) MarshalJSON() ([]byte, error) { - str := fmt.Sprintf(`"%f"`, v.F) - return []byte(str), nil +// MQTTDevicePropertyValue defines the value of the property. +// +kubebuilder:validation:Type="" +// +kubebuilder:validation:XPreserveUnknownFields +type MQTTDevicePropertyValue struct { + Raw []byte `json:"-"` } -func (v *ValueFloat) UnmarshalJSON(value []byte) error { - var err error - s := value - if len(s) > 0 && s[0] == '"' { - s = s[1:] + +// UnmarshalJSON implements the json.Unmarshaller interface. +func (in *MQTTDevicePropertyValue) UnmarshalJSON(data []byte) error { + if len(data) > 0 && string(data) != "null" { + in.Raw = data } - if len(s) > 0 && s[len(s)-1] == '"' { - s = s[:len(s)-1] + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (in MQTTDevicePropertyValue) MarshalJSON() ([]byte, error) { + if len(in.Raw) > 0 { + return in.Raw, nil } - v.F, err = strconv.ParseFloat(string(s), 64) - return err + return nil, nil } -type ValueArrayProps struct { - // +kubebuilder:validation:XPreserveUnknownFields - ValueProps `json:",inline"` +// OpenAPISchemaType is used by the kube-openapi generator when constructing +// the OpenAPI spec of this type. +// See: https://github.com/kubernetes/kube-openapi/tree/master/pkg/generators +func (MQTTDevicePropertyValue) OpenAPISchemaType() []string { + // TODO: return actual types when anyOf is supported + return nil } -type ValueProps struct { - // Reports the type of property. - ValueType ValueType `json:"valueType"` +// OpenAPISchemaFormat is used by the kube-openapi generator when constructing +// the OpenAPI spec of this type. +func (MQTTDevicePropertyValue) OpenAPISchemaFormat() string { return "" } - // Reports the value of int type. - // +optional - IntValue int64 `json:"intValue,omitempty"` +// MQTTDeviceProperty defines the specified property of MQTTDevice. +type MQTTDeviceProperty struct { + mqttapi.MQTTMessagePayloadOptions `json:",inline"` + mqttapi.MQTTMessageTopicOperation `json:",inline"` - // Reports the value of string type. + // Specifies the annotations of property. // +optional - StringValue string `json:"stringValue,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` - // Reports the value of float type. - // +optional - FloatValue *ValueFloat `json:"floatValue,omitempty"` + // Specifies the name of property. + // +kubebuilder:validation:Required + Name string `json:"name"` - // Reports the value of boolean type. + // Specifies the description of property. // +optional - BooleanValue bool `json:"booleanValue,omitempty"` + Description string `json:"description,omitempty"` - // Reports the value of array type. + // Specifies the type of property. + // +kubebuilder:validation:Required + Type MQTTDevicePropertyType `json:"type,omitempty"` + + // Specifies the value of property. // +optional - ArrayValue []ValueArrayProps `json:"arrayValue,omitempty"` + Value *MQTTDevicePropertyValue `json:"value,omitempty"` - // Reports the value of object type. - // +kubebuilder:validation:XPreserveUnknownFields + // Specifies the MIME of property value. // +optional - ObjectValue *runtime.RawExtension `json:"objectValue,omitempty"` -} + ContentType string `json:"contentType,omitempty"` -type Property struct { - SubInfo SubInfo `json:"subInfo"` - PubInfo PubInfo `json:"pubInfo,omitempty"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - JSONPath string `json:"jsonPath"` - Value ValueProps `json:"value,omitempty"` + // Specifies if the property is read-only. + // The default value is "true". + // +kubebuilder:default=true + ReadOnly *bool `json:"readOnly,omitempty"` } -type StatusProperty struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Value ValueProps `json:"value"` - UpdatedAt metav1.Time `json:"updateAt"` +// MQTTDeviceStatusProperty defines the observed property of MQTTDevice. +type MQTTDeviceStatusProperty struct { + MQTTDeviceProperty `json:",inline"` + + // Reports the updated timestamp of property. + // +optional + UpdatedAt *metav1.Time `json:"updateAt,omitempty"` } -// MqttDeviceSpec defines the desired state of MqttDevice -type MqttDeviceSpec struct { - Config MqttConfig `json:"config"` - Properties []Property `json:"properties"` +// MQTTDeviceSpec defines the desired state of MQTTDevice. +type MQTTDeviceSpec struct { + // Specifies the protocol for accessing the MQTT service. + // +kubebuilder:validation:Required + Protocol MQTTDeviceProtocol `json:"protocol"` + + // Specifies the properties of MQTTDevice. + // +listType=map + // +listMapKey=name + // +optional + Properties []MQTTDeviceProperty `json:"properties,omitempty"` } -// MqttDeviceStatus defines the observed state of MqttDevice -type MqttDeviceStatus struct { - Properties []StatusProperty `json:"properties"` +// MQTTDeviceStatus defines the observed state of MQTTDevice. +type MQTTDeviceStatus struct { + // Reports the properties of MQTTDevice. + // +optional + Properties []MQTTDeviceStatusProperty `json:"properties,omitempty"` } // +kubebuilder:object:root=true // +k8s:openapi-gen=true +// +kubebuilder:resource:shortName=mqtt // +kubebuilder:subresource:status -// MqttDevice is the Schema for the mqtt device API -type MqttDevice struct { +// +kubebuilder:printcolumn:name="PATTERN",type="string",JSONPath=`.spec.protocol.pattern` +// +kubebuilder:printcolumn:name="SERVER",type="string",JSONPath=`.spec.protocol.client.server` +// +kubebuilder:printcolumn:name="TOPIC",type="string",JSONPath=`.spec.protocol.message.topic` +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=`.metadata.creationTimestamp` +// MQTTDevice is the Schema for the MQTT device API. +type MQTTDevice struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec MqttDeviceSpec `json:"spec,omitempty"` - // +kubebuilder:validation:XPreserveUnknownFields - Status MqttDeviceStatus `json:"status,omitempty"` + Spec MQTTDeviceSpec `json:"spec,omitempty"` + Status MQTTDeviceStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true -// MqttDeviceList contains a list of MqttDevice -type MqttDeviceList struct { +// MQTTDeviceList contains a list of MQTTDevice. +type MQTTDeviceList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []MqttDevice `json:"items"` + + Items []MQTTDevice `json:"items"` } func init() { - SchemeBuilder.Register(&MqttDevice{}, &MqttDeviceList{}) + SchemeBuilder.Register(&MQTTDevice{}, &MQTTDeviceList{}) } diff --git a/adaptors/mqtt/api/v1alpha1/zz_generated.deepcopy.go b/adaptors/mqtt/api/v1alpha1/zz_generated.deepcopy.go index 7d7e0f7b..eb2e4e6d 100644 --- a/adaptors/mqtt/api/v1alpha1/zz_generated.deepcopy.go +++ b/adaptors/mqtt/api/v1alpha1/zz_generated.deepcopy.go @@ -20,26 +20,11 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/apimachinery/pkg/runtime" + runtime "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MqttConfig) DeepCopyInto(out *MqttConfig) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MqttConfig. -func (in *MqttConfig) DeepCopy() *MqttConfig { - if in == nil { - return nil - } - out := new(MqttConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MqttDevice) DeepCopyInto(out *MqttDevice) { +func (in *MQTTDevice) DeepCopyInto(out *MQTTDevice) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) @@ -47,18 +32,18 @@ func (in *MqttDevice) DeepCopyInto(out *MqttDevice) { in.Status.DeepCopyInto(&out.Status) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MqttDevice. -func (in *MqttDevice) DeepCopy() *MqttDevice { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MQTTDevice. +func (in *MQTTDevice) DeepCopy() *MQTTDevice { if in == nil { return nil } - out := new(MqttDevice) + out := new(MQTTDevice) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *MqttDevice) DeepCopyObject() runtime.Object { +func (in *MQTTDevice) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -66,31 +51,31 @@ func (in *MqttDevice) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MqttDeviceList) DeepCopyInto(out *MqttDeviceList) { +func (in *MQTTDeviceList) DeepCopyInto(out *MQTTDeviceList) { *out = *in out.TypeMeta = in.TypeMeta in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items - *out = make([]MqttDevice, len(*in)) + *out = make([]MQTTDevice, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MqttDeviceList. -func (in *MqttDeviceList) DeepCopy() *MqttDeviceList { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MQTTDeviceList. +func (in *MQTTDeviceList) DeepCopy() *MQTTDeviceList { if in == nil { return nil } - out := new(MqttDeviceList) + out := new(MQTTDeviceList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *MqttDeviceList) DeepCopyObject() runtime.Object { +func (in *MQTTDeviceList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -98,174 +83,156 @@ func (in *MqttDeviceList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MqttDeviceSpec) DeepCopyInto(out *MqttDeviceSpec) { +func (in *MQTTDeviceProperty) DeepCopyInto(out *MQTTDeviceProperty) { *out = *in - out.Config = in.Config - if in.Properties != nil { - in, out := &in.Properties, &out.Properties - *out = make([]Property, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) + in.MQTTMessagePayloadOptions.DeepCopyInto(&out.MQTTMessagePayloadOptions) + in.MQTTMessageTopicOperation.DeepCopyInto(&out.MQTTMessageTopicOperation) + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val } } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MqttDeviceSpec. -func (in *MqttDeviceSpec) DeepCopy() *MqttDeviceSpec { - if in == nil { - return nil + if in.Value != nil { + in, out := &in.Value, &out.Value + *out = new(MQTTDevicePropertyValue) + (*in).DeepCopyInto(*out) } - out := new(MqttDeviceSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MqttDeviceStatus) DeepCopyInto(out *MqttDeviceStatus) { - *out = *in - if in.Properties != nil { - in, out := &in.Properties, &out.Properties - *out = make([]StatusProperty, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + if in.ReadOnly != nil { + in, out := &in.ReadOnly, &out.ReadOnly + *out = new(bool) + **out = **in } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MqttDeviceStatus. -func (in *MqttDeviceStatus) DeepCopy() *MqttDeviceStatus { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MQTTDeviceProperty. +func (in *MQTTDeviceProperty) DeepCopy() *MQTTDeviceProperty { if in == nil { return nil } - out := new(MqttDeviceStatus) + out := new(MQTTDeviceProperty) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Property) DeepCopyInto(out *Property) { +func (in *MQTTDevicePropertyValue) DeepCopyInto(out *MQTTDevicePropertyValue) { *out = *in - out.SubInfo = in.SubInfo - out.PubInfo = in.PubInfo - in.Value.DeepCopyInto(&out.Value) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Property. -func (in *Property) DeepCopy() *Property { - if in == nil { - return nil + if in.Raw != nil { + in, out := &in.Raw, &out.Raw + *out = make([]byte, len(*in)) + copy(*out, *in) } - out := new(Property) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PubInfo) DeepCopyInto(out *PubInfo) { - *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PubInfo. -func (in *PubInfo) DeepCopy() *PubInfo { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MQTTDevicePropertyValue. +func (in *MQTTDevicePropertyValue) DeepCopy() *MQTTDevicePropertyValue { if in == nil { return nil } - out := new(PubInfo) + out := new(MQTTDevicePropertyValue) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StatusProperty) DeepCopyInto(out *StatusProperty) { +func (in *MQTTDeviceProtocol) DeepCopyInto(out *MQTTDeviceProtocol) { *out = *in - in.Value.DeepCopyInto(&out.Value) - in.UpdatedAt.DeepCopyInto(&out.UpdatedAt) + in.MQTTOptions.DeepCopyInto(&out.MQTTOptions) + if in.Schema != nil { + in, out := &in.Schema, &out.Schema + *out = new(MQTTDeviceSchema) + **out = **in + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatusProperty. -func (in *StatusProperty) DeepCopy() *StatusProperty { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MQTTDeviceProtocol. +func (in *MQTTDeviceProtocol) DeepCopy() *MQTTDeviceProtocol { if in == nil { return nil } - out := new(StatusProperty) + out := new(MQTTDeviceProtocol) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SubInfo) DeepCopyInto(out *SubInfo) { +func (in *MQTTDeviceSchema) DeepCopyInto(out *MQTTDeviceSchema) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubInfo. -func (in *SubInfo) DeepCopy() *SubInfo { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MQTTDeviceSchema. +func (in *MQTTDeviceSchema) DeepCopy() *MQTTDeviceSchema { if in == nil { return nil } - out := new(SubInfo) + out := new(MQTTDeviceSchema) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ValueArrayProps) DeepCopyInto(out *ValueArrayProps) { +func (in *MQTTDeviceSpec) DeepCopyInto(out *MQTTDeviceSpec) { *out = *in - in.ValueProps.DeepCopyInto(&out.ValueProps) + in.Protocol.DeepCopyInto(&out.Protocol) + if in.Properties != nil { + in, out := &in.Properties, &out.Properties + *out = make([]MQTTDeviceProperty, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ValueArrayProps. -func (in *ValueArrayProps) DeepCopy() *ValueArrayProps { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MQTTDeviceSpec. +func (in *MQTTDeviceSpec) DeepCopy() *MQTTDeviceSpec { if in == nil { return nil } - out := new(ValueArrayProps) + out := new(MQTTDeviceSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ValueFloat) DeepCopyInto(out *ValueFloat) { +func (in *MQTTDeviceStatus) DeepCopyInto(out *MQTTDeviceStatus) { *out = *in + if in.Properties != nil { + in, out := &in.Properties, &out.Properties + *out = make([]MQTTDeviceStatusProperty, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ValueFloat. -func (in *ValueFloat) DeepCopy() *ValueFloat { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MQTTDeviceStatus. +func (in *MQTTDeviceStatus) DeepCopy() *MQTTDeviceStatus { if in == nil { return nil } - out := new(ValueFloat) + out := new(MQTTDeviceStatus) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ValueProps) DeepCopyInto(out *ValueProps) { +func (in *MQTTDeviceStatusProperty) DeepCopyInto(out *MQTTDeviceStatusProperty) { *out = *in - if in.FloatValue != nil { - in, out := &in.FloatValue, &out.FloatValue - *out = new(ValueFloat) - **out = **in - } - if in.ArrayValue != nil { - in, out := &in.ArrayValue, &out.ArrayValue - *out = make([]ValueArrayProps, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.ObjectValue != nil { - in, out := &in.ObjectValue, &out.ObjectValue - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) + in.MQTTDeviceProperty.DeepCopyInto(&out.MQTTDeviceProperty) + if in.UpdatedAt != nil { + in, out := &in.UpdatedAt, &out.UpdatedAt + *out = (*in).DeepCopy() } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ValueProps. -func (in *ValueProps) DeepCopy() *ValueProps { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MQTTDeviceStatusProperty. +func (in *MQTTDeviceStatusProperty) DeepCopy() *MQTTDeviceStatusProperty { if in == nil { return nil } - out := new(ValueProps) + out := new(MQTTDeviceStatusProperty) in.DeepCopyInto(out) return out } diff --git a/adaptors/mqtt/deploy/e2e/all_in_one.yaml b/adaptors/mqtt/deploy/e2e/all_in_one.yaml index 70d35531..a95ab7f0 100644 --- a/adaptors/mqtt/deploy/e2e/all_in_one.yaml +++ b/adaptors/mqtt/deploy/e2e/all_in_one.yaml @@ -2,10 +2,13 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - devices.edge.cattle.io/description: "" - devices.edge.cattle.io/device-property: '{"name":"string","dataType":"string","value":"string","updatedAt":"date"}' + devices.edge.cattle.io/description: MQTT is a machine-to-machine (M2M)/"Internet + of Things" connectivity protocol, which was designed as an extremely lightweight + publish/subscribe messaging transport. The MQTT adaptor can assess and manage + the data stored in MQTT broker. + devices.edge.cattle.io/device-property: '{"name":"string","type":"string","value":"string","updatedAt":"date"}' devices.edge.cattle.io/enable: "true" - devices.edge.cattle.io/icon: "" + devices.edge.cattle.io/icon: http://mqtt.org/new/wp-content/uploads/2011/08/mqttorg-glow.png creationTimestamp: null labels: app.kubernetes.io/name: octopus-adaptor-mqtt @@ -14,16 +17,31 @@ metadata: spec: group: devices.edge.cattle.io names: - kind: MqttDevice - listKind: MqttDeviceList + kind: MQTTDevice + listKind: MQTTDeviceList plural: mqttdevices + shortNames: + - mqtt singular: mqttdevice scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .spec.protocol.pattern + name: PATTERN + type: string + - jsonPath: .spec.protocol.client.server + name: SERVER + type: string + - jsonPath: .spec.protocol.message.topic + name: TOPIC + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 schema: openAPIV3Schema: - description: MqttDevice is the Schema for the mqtt device API + description: MQTTDevice is the Schema for the MQTT device API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -38,173 +56,502 @@ spec: metadata: type: object spec: - description: MqttDeviceSpec defines the desired state of MqttDevice + description: MQTTDeviceSpec defines the desired state of MQTTDevice. properties: - config: - properties: - broker: - type: string - password: - type: string - username: - type: string - required: - - broker - type: object properties: + description: Specifies the properties of MQTTDevice. items: + description: MQTTDeviceProperty defines the specified property of + MQTTDevice. properties: - description: + annotations: + additionalProperties: + type: string + description: Specifies the annotations of property. + type: object + contentType: + description: Specifies the MIME of property value. type: string - jsonPath: + description: + description: Specifies the description of property. type: string name: + description: Specifies the name of property. type: string - pubInfo: - properties: - qos: - description: The qos type. - enum: - - 0 - - 1 - - 2 - type: integer - topic: - type: string - required: - - qos - - topic - type: object - subInfo: + operator: + description: Specifies the operator for rendering the `:operator` + keyword of topic. properties: - payloadType: - description: The payload type type. - enum: - - json + read: + description: Specifies the operator for rendering the `:operator` + keyword of topic during subscribing. type: string - qos: - description: The qos type. - enum: - - 0 - - 1 - - 2 - type: integer - topic: + write: + description: Specifies the operator for rendering the `:operator` + keyword of topic during publishing. type: string - required: - - payloadType - - qos - - topic type: object + path: + description: Specifies the path for rendering the `:path` keyword + of topic. + type: string + qos: + default: 1 + description: Specifies the QoS of the message. The default value + is "1". + enum: + - 0 + - 1 + - 2 + type: integer + readOnly: + default: true + description: Specifies if the property is read-only. The default + value is "true". + type: boolean + retained: + default: true + description: Specifies if the last published message to be retained. + The default value is "true". + type: boolean + type: + description: Specifies the type of property. + enum: + - int + - string + - float + - boolean + - object + - array + type: string value: - properties: - arrayValue: - description: Reports the value of array type. - items: - type: object - x-kubernetes-preserve-unknown-fields: true - type: array - booleanValue: - description: Reports the value of boolean type. - type: boolean - floatValue: - description: Reports the value of float type. - type: object - intValue: - description: Reports the value of int type. - format: int64 - type: integer - objectValue: - description: Reports the value of object type. - type: object - x-kubernetes-preserve-unknown-fields: true - stringValue: - description: Reports the value of string type. - type: string - valueType: - description: Reports the type of property. - enum: - - int - - string - - float - - boolean - - array - - object - type: string - required: - - valueType - type: object + description: Specifies the value of property. + x-kubernetes-preserve-unknown-fields: true required: - - jsonPath - name - - subInfo type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + protocol: + description: Specifies the protocol for accessing the MQTT service. + properties: + client: + description: Specifies the client settings. + properties: + autoReconnect: + default: true + description: Configures using the automatic reconnection logic. + The default value is "true". + type: boolean + basicAuth: + description: Specifies the username and password that the + client connects to the MQTT broker. Without the use of TLSConfig, + the account information will be sent in plaintext across + the wire. + properties: + password: + description: Specifies the password for basic authenication. + type: string + passwordRef: + description: Specifies the relationship of DeviceLink's + references to refer to the value as the password. + properties: + item: + description: Specifies the item name of the referred + reference. + type: string + name: + description: Specifies the name of reference. + type: string + required: + - item + - name + type: object + username: + description: Specifies the username for basic authentication. + type: string + usernameRef: + description: Specifies the relationship of DeviceLink's + references to refer to the value as the username. + properties: + item: + description: Specifies the item name of the referred + reference. + type: string + name: + description: Specifies the name of reference. + type: string + required: + - item + - name + type: object + type: object + cleanSession: + default: true + description: Specifies setting the "clean session" flag in + the connect message that the MQTT broker should not save + it. If the value is "false", the broker stores all missed + messages for the client that subscribed with QoS 1 or 2. + Any messages that were going to be sent by this client before + disconnecting previously but didn't send upon connecting + to the broker. The default value is "true". + type: boolean + connectTimeout: + default: 30s + description: Specifies the amount of time that the client + try to open a connection to an MQTT broker before timing + out and getting error. A duration of 0 never times out. + The default value is "30s". + type: string + disconnectQuiesce: + description: Specifies the quiesce when the client disconnects. + The default value is "5s". + type: string + httpHeaders: + additionalProperties: + items: + type: string + type: array + description: Specifies the additional HTTP headers that the + client sends in the WebSocket opening handshake. + type: object + x-kubernetes-map-type: atomic + keepAlive: + default: 30s + description: Specifies the amount of time that the client + should wait before sending a PING request to the broker. + This will allow the client to know that the connection has + not been lost with the server. A duration of 0 never keeps + alive. The default keep alive is "30s". + type: string + maxReconnectInterval: + default: 10m + description: Specifies the amount of time that the client + should wait before reconnecting to the broker. The first + reconnect interval is 1 second, and then the interval is + incremented by *2 until `MaxReconnectInterval` is reached. + This is only valid if `AutoReconnect` is true. A duration + of 0 may trigger the reconnection immediately. The default + value is "10m". + type: string + messageChannelDepth: + default: 100 + description: Specifies the size of the internal queue that + holds messages while the client is temporarily offline, + allowing the application to publish when the client is reconnected. + This is only valid if `AutoReconnect` is true. The default + value is "100". + type: integer + order: + default: true + description: Specifies the message routing to guarantee order + within each QoS level. If set to false, the message can + be delivered asynchronously from the client to the application + and possibly arrive out of order. The default value is "true". + type: boolean + pingTimeout: + default: 10s + description: Specifies the amount of time that the client + should wait after sending a PING request to the broker. + This will allow the client to know that the connection has + been lost with the server. A duration of 0 may cause unnecessary + timeout error. The default value is "10s". + type: string + protocolVersion: + default: 0 + description: Specifies the MQTT protocol version that the + cluster uses to connect to broker. Legitimate values are + currently 3 - MQTT v3.1 or 4 - MQTT v3.1.1. The default + value is 0, which means MQTT v3.1.1 identification is preferred. + enum: + - 0 + - 3 + - 4 + type: integer + resumeSubs: + default: false + description: Specifies to enable resuming of stored (un)subscribe + messages when connecting but not reconnecting. This is only + valid if `CleanSession` is false. The default value is "false". + type: boolean + server: + description: Specifies the server URI of MQTT broker, the + format should be `schema://host:port`. The "schema" is one + of the "ws", "wss", "tcp", "unix", "ssl", "tls" or "tcps". + pattern: ^(ws|wss|tcp|unix|ssl|tls|tcps)+://[^\s]*$ + type: string + store: + description: Specifies to provide message persistence in cases + where QoS level is 1 or 2. + properties: + directoryPrefix: + description: Specifies the directory prefix of the storage, + if using file store. The default value is "/var/run/octopus/mqtt". + pattern: ^/.*[^/]$ + type: string + type: + default: Memory + description: Specifies the type of storage. The default + value is "Memory". + enum: + - Memory + - File + type: string + type: object + tlsConfig: + description: Specifies the TLS configuration that the client + connects to the MQTT broker. + properties: + caFilePEM: + description: Specifies the PEM format content of the CA + certificate, which is used for validate the server certificate + with. + type: string + caFilePEMRef: + description: Specifies the relationship of DeviceLink's + references to refer to the value as the CA file PEM + content. + properties: + item: + description: Specifies the item name of the referred + reference. + type: string + name: + description: Specifies the name of reference. + type: string + required: + - item + - name + type: object + certFilePEM: + description: Specifies the PEM format content of the certificate(public + key), which is used for client authenticate to the server. + type: string + certFilePEMRef: + description: Specifies the relationship of DeviceLink's + references to refer to the value as the client certificate + file PEM content. + properties: + item: + description: Specifies the item name of the referred + reference. + type: string + name: + description: Specifies the name of reference. + type: string + required: + - item + - name + type: object + insecureSkipVerify: + description: Doesn't validate the server certificate. + type: boolean + keyFilePEM: + description: Specifies the PEM format content of the key(private + key), which is used for client authenticate to the server. + type: string + keyFilePEMRef: + description: Specifies the relationship of DeviceLink's + references to refer to the value as the client key file + PEM content. + properties: + item: + description: Specifies the item name of the referred + reference. + type: string + name: + description: Specifies the name of reference. + type: string + required: + - item + - name + type: object + serverName: + description: Indicates the name of the server, ref to + http://tools.ietf.org/html/rfc4366#section-3.1. + type: string + type: object + waitTimeout: + description: Specifies the amount of time that the client + should timeout after subscribed/published a message. A duration + of 0 never times out. + type: string + writeTimeout: + default: 30s + description: Specifies the amount of time that the client + publish a message successfully before getting a timeout + error. A duration of 0 never times out. The default value + is "30s". + type: string + required: + - server + type: object + message: + description: Specifies the message settings. + properties: + operator: + description: Specifies the operator for rendering the `:operator` + keyword of topic. + properties: + read: + description: Specifies the operator for rendering the + `:operator` keyword of topic during subscribing. + type: string + write: + description: Specifies the operator for rendering the + `:operator` keyword of topic during publishing. + type: string + type: object + path: + description: Specifies the path for rendering the `:path` + keyword of topic. + type: string + qos: + default: 1 + description: Specifies the QoS of the message. The default + value is "1". + enum: + - 0 + - 1 + - 2 + type: integer + retained: + default: true + description: Specifies if the last published message to be + retained. The default value is "true". + type: boolean + topic: + description: Specifies the topic. + pattern: .*[^/]$ + type: string + will: + description: Specifies the will message. + properties: + content: + description: Specifies the content of will message. The + serialized form of the content is a base64 encoded string, + representing the arbitrary (possibly non-string) content + value here. + type: string + topic: + description: Specifies the topic of will message. if not + set, the topic will append "$will" to the topic name + specified in parent field as its topic name. + pattern: .*[^/]$ + type: string + required: + - content + type: object + required: + - topic + type: object + pattern: + description: Specifies the pattern of MQTTDevice protocol. + enum: + - AttributedMessage + - AttributedTopic + type: string + schema: + description: Specifies the schema of the pattern. + properties: + reference: + description: Specifies the reference for schema. + type: string + type: + description: Specifies the type of schema. + type: string + type: object + required: + - client + - message + - pattern + type: object required: - - config - - properties + - protocol type: object status: - description: MqttDeviceStatus defines the observed state of MqttDevice + description: MQTTDeviceStatus defines the observed state of MQTTDevice. properties: properties: + description: Reports the properties of MQTTDevice. items: + description: MQTTDeviceStatusProperty defines the observed property + of MQTTDevice. properties: + annotations: + additionalProperties: + type: string + description: Specifies the annotations of property. + type: object + contentType: + description: Specifies the MIME of property value. + type: string description: + description: Specifies the description of property. type: string name: + description: Specifies the name of property. type: string - updateAt: - format: date-time - type: string - value: + operator: + description: Specifies the operator for rendering the `:operator` + keyword of topic. properties: - arrayValue: - description: Reports the value of array type. - items: - type: object - x-kubernetes-preserve-unknown-fields: true - type: array - booleanValue: - description: Reports the value of boolean type. - type: boolean - floatValue: - description: Reports the value of float type. - type: object - intValue: - description: Reports the value of int type. - format: int64 - type: integer - objectValue: - description: Reports the value of object type. - type: object - x-kubernetes-preserve-unknown-fields: true - stringValue: - description: Reports the value of string type. + read: + description: Specifies the operator for rendering the `:operator` + keyword of topic during subscribing. type: string - valueType: - description: Reports the type of property. - enum: - - int - - string - - float - - boolean - - array - - object + write: + description: Specifies the operator for rendering the `:operator` + keyword of topic during publishing. type: string - required: - - valueType type: object + path: + description: Specifies the path for rendering the `:path` keyword + of topic. + type: string + qos: + default: 1 + description: Specifies the QoS of the message. The default value + is "1". + enum: + - 0 + - 1 + - 2 + type: integer + readOnly: + default: true + description: Specifies if the property is read-only. The default + value is "true". + type: boolean + retained: + default: true + description: Specifies if the last published message to be retained. + The default value is "true". + type: boolean + type: + description: Specifies the type of property. + enum: + - int + - string + - float + - boolean + - object + - array + type: string + updateAt: + description: Reports the updated timestamp of property. + format: date-time + type: string + value: + description: Specifies the value of property. + x-kubernetes-preserve-unknown-fields: true required: - name - - updateAt - - value type: object type: array - required: - - properties type: object - x-kubernetes-preserve-unknown-fields: true type: object served: true storage: true diff --git a/adaptors/mqtt/deploy/e2e/dl_attributed_message_bedroom_light.yaml b/adaptors/mqtt/deploy/e2e/dl_attributed_message_bedroom_light.yaml new file mode 100644 index 00000000..6a5808f3 --- /dev/null +++ b/adaptors/mqtt/deploy/e2e/dl_attributed_message_bedroom_light.yaml @@ -0,0 +1,76 @@ +apiVersion: edge.cattle.io/v1alpha1 +kind: DeviceLink +metadata: + name: bedroom-light +spec: + adaptor: + node: edge-worker + name: adaptors.edge.cattle.io/mqtt + model: + apiVersion: "devices.edge.cattle.io/v1alpha1" + kind: "MQTTDevice" + template: + metadata: + labels: + device: bedroom-light + spec: + protocol: + pattern: "AttributedMessage" + client: + server: "tcp://test.mosquitto.org:1883" + message: + topic: "cattle.io/octopus/home/bedroom/light/:operator" + operator: + write: "set" + properties: + - name: "switch" + path: "switch" + description: "The switch of light" + type: "boolean" + readOnly: false + - name: "gear" + path: "action.gear" + description: "The gear of light" + type: "string" + readOnly: false + annotations: + type: "enum" + format: "low,mid,high" + - name: "power" + path: "parameter.power" + description: "The power of light" + type: "float" + annotations: + group: "parameter" + unit: "watt" + - name: "luminance" + path: "parameter.luminance" + description: "The luminance of light" + type: "int" + annotations: + group: "parameter" + unit: "luminance" + - name: "manufacturer" + path: "production.manufacturer" + description: "The manufacturer of light" + type: "string" + annotations: + group: "production" + - name: "productionDate" + path: "production.date" + description: "The production date of light" + type: "string" + annotations: + group: "production" + type: "datetime" + standard: "ISO 8601" + format: "YYYY-MM-DDThh:mm:ss.SSZ" + - name: "serviceLife" + path: "production.serviceLife" + description: "The service life of light" + type: "string" + annotations: + group: "production" + type: "duration" + standard: "ISO 8601" + format: "PYYMMDD" diff --git a/adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_door.yaml b/adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_door.yaml new file mode 100644 index 00000000..37a65b1d --- /dev/null +++ b/adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_door.yaml @@ -0,0 +1,75 @@ +apiVersion: edge.cattle.io/v1alpha1 +kind: DeviceLink +metadata: + name: kitchen-door +spec: + adaptor: + node: edge-worker + name: adaptors.edge.cattle.io/mqtt + model: + apiVersion: "devices.edge.cattle.io/v1alpha1" + kind: "MQTTDevice" + template: + metadata: + labels: + device: kitchen-door + spec: + protocol: + pattern: "AttributedTopic" + client: + server: "tcp://test.mosquitto.org:1883" + message: + topic: "cattle.io/octopus/home/status/kitchen/door/:path" + properties: + - name: "state" + path: "state" + description: "The state of door" + type: "string" + annotations: + type: "enum" + format: "open,close" + - name: "width" + path: "width" + description: "The width of door" + type: "float" + annotations: + unit: "meter" + - name: "height" + path: "height" + description: "The height of door" + type: "float" + annotations: + unit: "meter" + - name: "material" + path: "material" + description: "The material of light" + type: "string" + # status: + # properties: + # - name: "state" + # path: "state" + # description: "The state of door" + # type: "string" + # value: "open" + # annotations: + # type: "enum" + # format: "open,close" + # - name: "width" + # path: "width" + # description: "The width of door" + # type: "float" + # value: "1.2" + # annotations: + # unit: "meter" + # - name: "height" + # path: "height" + # description: "The height of door" + # type: "float" + # value: "1.8" + # annotations: + # unit: "meter" + # - name: "material" + # path: "material" + # description: "The material of light" + # type: "string" + # value: "wood" diff --git a/adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_light.yaml b/adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_light.yaml new file mode 100644 index 00000000..e77dd987 --- /dev/null +++ b/adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_light.yaml @@ -0,0 +1,137 @@ +apiVersion: edge.cattle.io/v1alpha1 +kind: DeviceLink +metadata: + name: kitchen-light +spec: + adaptor: + node: edge-worker + name: adaptors.edge.cattle.io/mqtt + model: + apiVersion: "devices.edge.cattle.io/v1alpha1" + kind: "MQTTDevice" + template: + metadata: + labels: + device: kitchen-light + spec: + protocol: + pattern: "AttributedTopic" + client: + server: "tcp://test.mosquitto.org:1883" + message: + topic: "cattle.io/octopus/home/:operator/kitchen/light/:path" + operator: + read: "status" + write: "set" + properties: + - name: "switch" + path: "switch" + description: "The switch of light" + type: "boolean" + readOnlye: false + qos: 2 + - name: "gear" + path: "gear" + description: "The gear of light" + type: "string" + readOnlye: false + annotations: + type: "enum" + format: "low,mid,high" + - name: "power" + path: "parameter_power" + description: "The power of light" + type: "float" + annotations: + group: "parameter" + unit: "watt" + - name: "luminance" + path: "parameter_luminance" + description: "The luminance of light" + type: "int" + annotations: + group: "parameter" + unit: "luminance" + - name: "manufacturer" + description: "The manufacturer of light" + type: "string" + annotations: + group: "production" + - name: "productionDate" + path: "production_date" + description: "The production date of light" + type: "string" + annotations: + group: "production" + type: "datetime" + standard: "ISO 8601" + format: "YYYY-MM-DDThh:mm:ss.SSZ" + - name: "serviceLife" + path: "service_life" + description: "The service life of light" + type: "string" + annotations: + group: "production" + type: "duration" + standard: "ISO 8601" + format: "PYYMMDD" + # status: + # properties: + # - name: "switch" + # path: "switch" + # description: "The switch of light" + # type: "boolean" + # value: "false" + # readOnlye: false + # qos: 2 + # - name: "gear" + # path: "gear" + # description: "The gear of light" + # type: "string" + # value: "low" + # readOnlye: false + # annotations: + # type: "enum" + # format: "low,mid,high" + # - name: "power" + # path: "parameter_power" + # description: "The power of light" + # type: "float" + # value: "3.0" + # annotations: + # group: "parameter" + # unit: "watt" + # - name: "luminance" + # path: "parameter_luminance" + # description: "The luminance of light" + # type: "int" + # value: "245" + # annotations: + # group: "parameter" + # unit: "luminance" + # - name: "manufacturer" + # description: "The manufacturer of light" + # type: "string" + # value: "Rancher Octopus Fake Device" + # annotations: + # group: "production" + # - name: "productionDate" + # path: "production_date" + # description: "The production date of light" + # type: "string" + # value: "2020-07-08T13:24:00.00Z" + # annotations: + # group: "production" + # type: "datetime" + # standard: "ISO 8601" + # format: "YYYY-MM-DDThh:mm:ss.SSZ" + # - name: "serviceLife" + # path: "service_life" + # description: "The service life of light" + # type: "string" + # value: "P1Y0M0D" + # annotations: + # group: "production" + # type: "duration" + # standard: "ISO 8601" + # format: "PYYMMDD" diff --git a/adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_monitor.yaml b/adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_monitor.yaml new file mode 100644 index 00000000..325f87f0 --- /dev/null +++ b/adaptors/mqtt/deploy/e2e/dl_attributed_topic_kitchen_monitor.yaml @@ -0,0 +1,50 @@ +apiVersion: edge.cattle.io/v1alpha1 +kind: DeviceLink +metadata: + name: kitchen-monitor +spec: + adaptor: + node: edge-worker + name: adaptors.edge.cattle.io/mqtt + model: + apiVersion: "devices.edge.cattle.io/v1alpha1" + kind: "MQTTDevice" + template: + metadata: + labels: + device: kitchen-monitor + spec: + protocol: + pattern: "AttributedTopic" + client: + server: "tcp://test.mosquitto.org:1883" + message: + topic: "cattle.io/octopus/home/status/kitchen/:path" + properties: + - name: "doorState" + path: "door/state" + description: "The state of door" + type: "string" + annotations: + type: "enum" + format: "open,close" + - name: "isLightOn" + description: "The state of light" + path: "light/switch" + type: "boolean" + # status: + # properties: + # - name: "doorState" + # path: "door/state" + # description: "The state of door" + # type: "string" + # value: "open" + # annotations: + # type: "enum" + # format: "open,close" + # - name: "isLightOn" + # description: "The state of light" + # path: "light/switch" + # type: "boolean" + # value: "false" + diff --git a/adaptors/mqtt/deploy/e2e/dl_attributed_topic_livingroom_light.yaml b/adaptors/mqtt/deploy/e2e/dl_attributed_topic_livingroom_light.yaml new file mode 100644 index 00000000..73f69fc2 --- /dev/null +++ b/adaptors/mqtt/deploy/e2e/dl_attributed_topic_livingroom_light.yaml @@ -0,0 +1,82 @@ +apiVersion: edge.cattle.io/v1alpha1 +kind: DeviceLink +metadata: + name: living-room-light +spec: + adaptor: + node: edge-worker + name: adaptors.edge.cattle.io/mqtt + model: + apiVersion: "devices.edge.cattle.io/v1alpha1" + kind: "MQTTDevice" + template: + metadata: + labels: + device: living-room-light + spec: + protocol: + pattern: "AttributedTopic" + client: + server: "tcp://test.mosquitto.org:1883" + message: + topic: "cattle.io/octopus/home/livingroom/light/:path/:operator" + operator: + read: "" + write: "set" + properties: + - name: "switch" + path: "switch" + description: "The switch of light" + type: "boolean" + readOnly: false + - name: "gear" + path: "gear" + description: "The gear of light" + type: "string" + readOnly: false + annotations: + type: "enum" + format: "low,mid,high" + - name: "parameter" + path: "parameter" + description: "The parameter information of light" + type: "array" + - name: "production" + path: "production" + description: "The production information of light" + type: "object" + # status: + # properties: + # - name: "switch" + # path: "switch" + # description: "The switch of light" + # type: "boolean" + # value: "open" + # readOnly: false + # - name: "gear" + # path: "gear" + # description: "The gear of light" + # type: "string" + # value: "mid" + # readOnly: false + # annotations: + # type: "enum" + # format: "low,mid,high" + # - name: "parameter" + # path: "parameter" + # description: "The parameter information of light" + # type: "array" + # value: + # - name: power + # value: 70.0w + # - name: luminance + # value: 4900lm + # - name: "production" + # path: "production" + # description: "The production information of light" + # type: "object" + # value: + # manufacturer: "Rancher Octopus Fake Device" + # date: "2020-07-09T13:00:00.00Z" + # serviceLife: "P1Y0M0D" + diff --git a/adaptors/mqtt/deploy/e2e/roomlightcase1.yaml b/adaptors/mqtt/deploy/e2e/roomlightcase1.yaml deleted file mode 100644 index d1aa9b80..00000000 --- a/adaptors/mqtt/deploy/e2e/roomlightcase1.yaml +++ /dev/null @@ -1,89 +0,0 @@ -apiVersion: edge.cattle.io/v1alpha1 -kind: DeviceLink -metadata: - name: mqtt-test -spec: - adaptor: - node: k3d-k3s-default-server - name: adaptors.edge.cattle.io/mqtt - model: - apiVersion: "devices.edge.cattle.io/v1alpha1" - kind: "MqttDevice" - template: - metadata: - labels: - device: mqtt-test - spec: - config: - broker: "tcp://192.168.8.246:1883" - password: parchk123 - username: parchk - properties: - - name: "switch" - description: "the room light switch" - jsonPath: "switch" - subInfo: - topic: "device/room/light" - payloadType: "json" - qos: 2 - pubInfo: - topic: "device/room/light/cmd" - qos: 2 - - name: "brightness" - description: "the room light brightness" - jsonPath: "brightness" - subInfo: - topic: "device/room/light" - payloadType: "json" - qos: 2 - pubInfo: - topic: "device/room/light/cmd" - qos: 2 - - name: "power" - description: "the room light power" - jsonPath: "power" - subInfo: - topic: "device/room/light" - payloadType: "json" - qos: 2 - pubInfo: - topic: "device/room/light/cmd" - qos: 2 - - name: "attr" - description: "the room light attr" - jsonPath: "attr" - subInfo: - topic: "device/room/light" - payloadType: "json" - qos: 2 - pubInfo: - topic: "device/room/light/cmd" - qos: 2 - -# case: -# if the "device/room/light" topic subscribe get value : {"switch":"off","brightness"：4,"power":{"powerDissipation":"10KWH","electricQuantity":19.99}}, -# we will get the status like this: -# status: -# properties: -# - description: "the room light switch" -# name: "switch" -# updateAt: "2020-05-20T06:52:41Z" -# value: -# stringValue: "off" -# type: "string" -# - description: "the room light brightness" -# name: "brightness" -# updateAt: "2020-05-20T06:59:59Z" -# value: -# intValue: "4" -# type: "int" -# - description: "the room light power" -# name: "power" -# updateAt: "2020-05-20T06:49:48Z" -# value: -# objectValue: -# electricQuantity: 19.99 -# powerDissipation: "10KWH" -# type: "object" - - diff --git a/adaptors/mqtt/deploy/manifests/crd/base/devices.edge.cattle.io_mqttdevices.yaml b/adaptors/mqtt/deploy/manifests/crd/base/devices.edge.cattle.io_mqttdevices.yaml index 0b8a68fd..ff8e7a4c 100644 --- a/adaptors/mqtt/deploy/manifests/crd/base/devices.edge.cattle.io_mqttdevices.yaml +++ b/adaptors/mqtt/deploy/manifests/crd/base/devices.edge.cattle.io_mqttdevices.yaml @@ -10,16 +10,31 @@ metadata: spec: group: devices.edge.cattle.io names: - kind: MqttDevice - listKind: MqttDeviceList + kind: MQTTDevice + listKind: MQTTDeviceList plural: mqttdevices + shortNames: + - mqtt singular: mqttdevice scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .spec.protocol.pattern + name: PATTERN + type: string + - jsonPath: .spec.protocol.client.server + name: SERVER + type: string + - jsonPath: .spec.protocol.message.topic + name: TOPIC + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 schema: openAPIV3Schema: - description: MqttDevice is the Schema for the mqtt device API + description: MQTTDevice is the Schema for the MQTT device API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -34,173 +49,502 @@ spec: metadata: type: object spec: - description: MqttDeviceSpec defines the desired state of MqttDevice + description: MQTTDeviceSpec defines the desired state of MQTTDevice. properties: - config: - properties: - broker: - type: string - password: - type: string - username: - type: string - required: - - broker - type: object properties: + description: Specifies the properties of MQTTDevice. items: + description: MQTTDeviceProperty defines the specified property of + MQTTDevice. properties: - description: + annotations: + additionalProperties: + type: string + description: Specifies the annotations of property. + type: object + contentType: + description: Specifies the MIME of property value. type: string - jsonPath: + description: + description: Specifies the description of property. type: string name: + description: Specifies the name of property. type: string - pubInfo: - properties: - qos: - description: The qos type. - enum: - - 0 - - 1 - - 2 - type: integer - topic: - type: string - required: - - qos - - topic - type: object - subInfo: + operator: + description: Specifies the operator for rendering the `:operator` + keyword of topic. properties: - payloadType: - description: The payload type type. - enum: - - json + read: + description: Specifies the operator for rendering the `:operator` + keyword of topic during subscribing. type: string - qos: - description: The qos type. - enum: - - 0 - - 1 - - 2 - type: integer - topic: + write: + description: Specifies the operator for rendering the `:operator` + keyword of topic during publishing. type: string - required: - - payloadType - - qos - - topic type: object + path: + description: Specifies the path for rendering the `:path` keyword + of topic. + type: string + qos: + default: 1 + description: Specifies the QoS of the message. The default value + is "1". + enum: + - 0 + - 1 + - 2 + type: integer + readOnly: + default: true + description: Specifies if the property is read-only. The default + value is "true". + type: boolean + retained: + default: true + description: Specifies if the last published message to be retained. + The default value is "true". + type: boolean + type: + description: Specifies the type of property. + enum: + - int + - string + - float + - boolean + - object + - array + type: string value: - properties: - arrayValue: - description: Reports the value of array type. - items: - type: object - x-kubernetes-preserve-unknown-fields: true - type: array - booleanValue: - description: Reports the value of boolean type. - type: boolean - floatValue: - description: Reports the value of float type. - type: object - intValue: - description: Reports the value of int type. - format: int64 - type: integer - objectValue: - description: Reports the value of object type. - type: object - x-kubernetes-preserve-unknown-fields: true - stringValue: - description: Reports the value of string type. - type: string - valueType: - description: Reports the type of property. - enum: - - int - - string - - float - - boolean - - array - - object - type: string - required: - - valueType - type: object + description: Specifies the value of property. + x-kubernetes-preserve-unknown-fields: true required: - - jsonPath - name - - subInfo type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + protocol: + description: Specifies the protocol for accessing the MQTT service. + properties: + client: + description: Specifies the client settings. + properties: + autoReconnect: + default: true + description: Configures using the automatic reconnection logic. + The default value is "true". + type: boolean + basicAuth: + description: Specifies the username and password that the + client connects to the MQTT broker. Without the use of TLSConfig, + the account information will be sent in plaintext across + the wire. + properties: + password: + description: Specifies the password for basic authenication. + type: string + passwordRef: + description: Specifies the relationship of DeviceLink's + references to refer to the value as the password. + properties: + item: + description: Specifies the item name of the referred + reference. + type: string + name: + description: Specifies the name of reference. + type: string + required: + - item + - name + type: object + username: + description: Specifies the username for basic authentication. + type: string + usernameRef: + description: Specifies the relationship of DeviceLink's + references to refer to the value as the username. + properties: + item: + description: Specifies the item name of the referred + reference. + type: string + name: + description: Specifies the name of reference. + type: string + required: + - item + - name + type: object + type: object + cleanSession: + default: true + description: Specifies setting the "clean session" flag in + the connect message that the MQTT broker should not save + it. If the value is "false", the broker stores all missed + messages for the client that subscribed with QoS 1 or 2. + Any messages that were going to be sent by this client before + disconnecting previously but didn't send upon connecting + to the broker. The default value is "true". + type: boolean + connectTimeout: + default: 30s + description: Specifies the amount of time that the client + try to open a connection to an MQTT broker before timing + out and getting error. A duration of 0 never times out. + The default value is "30s". + type: string + disconnectQuiesce: + description: Specifies the quiesce when the client disconnects. + The default value is "5s". + type: string + httpHeaders: + additionalProperties: + items: + type: string + type: array + description: Specifies the additional HTTP headers that the + client sends in the WebSocket opening handshake. + type: object + x-kubernetes-map-type: atomic + keepAlive: + default: 30s + description: Specifies the amount of time that the client + should wait before sending a PING request to the broker. + This will allow the client to know that the connection has + not been lost with the server. A duration of 0 never keeps + alive. The default keep alive is "30s". + type: string + maxReconnectInterval: + default: 10m + description: Specifies the amount of time that the client + should wait before reconnecting to the broker. The first + reconnect interval is 1 second, and then the interval is + incremented by *2 until `MaxReconnectInterval` is reached. + This is only valid if `AutoReconnect` is true. A duration + of 0 may trigger the reconnection immediately. The default + value is "10m". + type: string + messageChannelDepth: + default: 100 + description: Specifies the size of the internal queue that + holds messages while the client is temporarily offline, + allowing the application to publish when the client is reconnected. + This is only valid if `AutoReconnect` is true. The default + value is "100". + type: integer + order: + default: true + description: Specifies the message routing to guarantee order + within each QoS level. If set to false, the message can + be delivered asynchronously from the client to the application + and possibly arrive out of order. The default value is "true". + type: boolean + pingTimeout: + default: 10s + description: Specifies the amount of time that the client + should wait after sending a PING request to the broker. + This will allow the client to know that the connection has + been lost with the server. A duration of 0 may cause unnecessary + timeout error. The default value is "10s". + type: string + protocolVersion: + default: 0 + description: Specifies the MQTT protocol version that the + cluster uses to connect to broker. Legitimate values are + currently 3 - MQTT v3.1 or 4 - MQTT v3.1.1. The default + value is 0, which means MQTT v3.1.1 identification is preferred. + enum: + - 0 + - 3 + - 4 + type: integer + resumeSubs: + default: false + description: Specifies to enable resuming of stored (un)subscribe + messages when connecting but not reconnecting. This is only + valid if `CleanSession` is false. The default value is "false". + type: boolean + server: + description: Specifies the server URI of MQTT broker, the + format should be `schema://host:port`. The "schema" is one + of the "ws", "wss", "tcp", "unix", "ssl", "tls" or "tcps". + pattern: ^(ws|wss|tcp|unix|ssl|tls|tcps)+://[^\s]*$ + type: string + store: + description: Specifies to provide message persistence in cases + where QoS level is 1 or 2. + properties: + directoryPrefix: + description: Specifies the directory prefix of the storage, + if using file store. The default value is "/var/run/octopus/mqtt". + pattern: ^/.*[^/]$ + type: string + type: + default: Memory + description: Specifies the type of storage. The default + value is "Memory". + enum: + - Memory + - File + type: string + type: object + tlsConfig: + description: Specifies the TLS configuration that the client + connects to the MQTT broker. + properties: + caFilePEM: + description: Specifies the PEM format content of the CA + certificate, which is used for validate the server certificate + with. + type: string + caFilePEMRef: + description: Specifies the relationship of DeviceLink's + references to refer to the value as the CA file PEM + content. + properties: + item: + description: Specifies the item name of the referred + reference. + type: string + name: + description: Specifies the name of reference. + type: string + required: + - item + - name + type: object + certFilePEM: + description: Specifies the PEM format content of the certificate(public + key), which is used for client authenticate to the server. + type: string + certFilePEMRef: + description: Specifies the relationship of DeviceLink's + references to refer to the value as the client certificate + file PEM content. + properties: + item: + description: Specifies the item name of the referred + reference. + type: string + name: + description: Specifies the name of reference. + type: string + required: + - item + - name + type: object + insecureSkipVerify: + description: Doesn't validate the server certificate. + type: boolean + keyFilePEM: + description: Specifies the PEM format content of the key(private + key), which is used for client authenticate to the server. + type: string + keyFilePEMRef: + description: Specifies the relationship of DeviceLink's + references to refer to the value as the client key file + PEM content. + properties: + item: + description: Specifies the item name of the referred + reference. + type: string + name: + description: Specifies the name of reference. + type: string + required: + - item + - name + type: object + serverName: + description: Indicates the name of the server, ref to + http://tools.ietf.org/html/rfc4366#section-3.1. + type: string + type: object + waitTimeout: + description: Specifies the amount of time that the client + should timeout after subscribed/published a message. A duration + of 0 never times out. + type: string + writeTimeout: + default: 30s + description: Specifies the amount of time that the client + publish a message successfully before getting a timeout + error. A duration of 0 never times out. The default value + is "30s". + type: string + required: + - server + type: object + message: + description: Specifies the message settings. + properties: + operator: + description: Specifies the operator for rendering the `:operator` + keyword of topic. + properties: + read: + description: Specifies the operator for rendering the + `:operator` keyword of topic during subscribing. + type: string + write: + description: Specifies the operator for rendering the + `:operator` keyword of topic during publishing. + type: string + type: object + path: + description: Specifies the path for rendering the `:path` + keyword of topic. + type: string + qos: + default: 1 + description: Specifies the QoS of the message. The default + value is "1". + enum: + - 0 + - 1 + - 2 + type: integer + retained: + default: true + description: Specifies if the last published message to be + retained. The default value is "true". + type: boolean + topic: + description: Specifies the topic. + pattern: .*[^/]$ + type: string + will: + description: Specifies the will message. + properties: + content: + description: Specifies the content of will message. The + serialized form of the content is a base64 encoded string, + representing the arbitrary (possibly non-string) content + value here. + type: string + topic: + description: Specifies the topic of will message. if not + set, the topic will append "$will" to the topic name + specified in parent field as its topic name. + pattern: .*[^/]$ + type: string + required: + - content + type: object + required: + - topic + type: object + pattern: + description: Specifies the pattern of MQTTDevice protocol. + enum: + - AttributedMessage + - AttributedTopic + type: string + schema: + description: Specifies the schema of the pattern. + properties: + reference: + description: Specifies the reference for schema. + type: string + type: + description: Specifies the type of schema. + type: string + type: object + required: + - client + - message + - pattern + type: object required: - - config - - properties + - protocol type: object status: - description: MqttDeviceStatus defines the observed state of MqttDevice + description: MQTTDeviceStatus defines the observed state of MQTTDevice. properties: properties: + description: Reports the properties of MQTTDevice. items: + description: MQTTDeviceStatusProperty defines the observed property + of MQTTDevice. properties: + annotations: + additionalProperties: + type: string + description: Specifies the annotations of property. + type: object + contentType: + description: Specifies the MIME of property value. + type: string description: + description: Specifies the description of property. type: string name: + description: Specifies the name of property. type: string - updateAt: - format: date-time - type: string - value: + operator: + description: Specifies the operator for rendering the `:operator` + keyword of topic. properties: - arrayValue: - description: Reports the value of array type. - items: - type: object - x-kubernetes-preserve-unknown-fields: true - type: array - booleanValue: - description: Reports the value of boolean type. - type: boolean - floatValue: - description: Reports the value of float type. - type: object - intValue: - description: Reports the value of int type. - format: int64 - type: integer - objectValue: - description: Reports the value of object type. - type: object - x-kubernetes-preserve-unknown-fields: true - stringValue: - description: Reports the value of string type. + read: + description: Specifies the operator for rendering the `:operator` + keyword of topic during subscribing. type: string - valueType: - description: Reports the type of property. - enum: - - int - - string - - float - - boolean - - array - - object + write: + description: Specifies the operator for rendering the `:operator` + keyword of topic during publishing. type: string - required: - - valueType type: object + path: + description: Specifies the path for rendering the `:path` keyword + of topic. + type: string + qos: + default: 1 + description: Specifies the QoS of the message. The default value + is "1". + enum: + - 0 + - 1 + - 2 + type: integer + readOnly: + default: true + description: Specifies if the property is read-only. The default + value is "true". + type: boolean + retained: + default: true + description: Specifies if the last published message to be retained. + The default value is "true". + type: boolean + type: + description: Specifies the type of property. + enum: + - int + - string + - float + - boolean + - object + - array + type: string + updateAt: + description: Reports the updated timestamp of property. + format: date-time + type: string + value: + description: Specifies the value of property. + x-kubernetes-preserve-unknown-fields: true required: - name - - updateAt - - value type: object type: array - required: - - properties type: object - x-kubernetes-preserve-unknown-fields: true type: object served: true storage: true diff --git a/adaptors/mqtt/deploy/manifests/crd/kustomization.yaml b/adaptors/mqtt/deploy/manifests/crd/kustomization.yaml index f25521ec..4224edb3 100644 --- a/adaptors/mqtt/deploy/manifests/crd/kustomization.yaml +++ b/adaptors/mqtt/deploy/manifests/crd/kustomization.yaml @@ -1,8 +1,8 @@ commonAnnotations: - devices.edge.cattle.io/enable: "true" # this is required - devices.edge.cattle.io/device-property: '{"name":"string","dataType":"string","value":"string","updatedAt":"date"}' # specify device property to be displayed in the UI - devices.edge.cattle.io/icon: "" # define the icon for the Edge UI - devices.edge.cattle.io/description: "" # add your custom device adaptor description + devices.edge.cattle.io/enable: "true" + devices.edge.cattle.io/device-property: '{"name":"string","type":"string","value":"string","updatedAt":"date"}' + devices.edge.cattle.io/icon: "http://mqtt.org/new/wp-content/uploads/2011/08/mqttorg-glow.png" + devices.edge.cattle.io/description: 'MQTT is a machine-to-machine (M2M)/"Internet of Things" connectivity protocol, which was designed as an extremely lightweight publish/subscribe messaging transport. The MQTT adaptor can assess and manage the data stored in MQTT broker.' resources: - base/devices.edge.cattle.io_mqttdevices.yaml diff --git a/adaptors/mqtt/hack/lib/constant.sh b/adaptors/mqtt/hack/lib/constant.sh old mode 100755 new mode 100644 diff --git a/adaptors/mqtt/hack/make-rules/adaptor.sh b/adaptors/mqtt/hack/make-rules/adaptor.sh index 3dde41e7..adff09b5 100755 --- a/adaptors/mqtt/hack/make-rules/adaptor.sh +++ b/adaptors/mqtt/hack/make-rules/adaptor.sh @@ -18,6 +18,8 @@ function generate() { octopus::log::info "generating adaptor ${adaptor}..." + # TODO adjust the generation logic if needed + octopus::log::info "generating objects" rm -f "${CURR_DIR}/api/*/zz_generated*" octopus::controller_gen::generate \ @@ -82,6 +84,7 @@ function build() { octopus::log::info "building adaptor ${adaptor}(${GIT_VERSION},${GIT_COMMIT},${GIT_TREE_STATE},${BUILD_DATE})..." + # TODO adjust the ldflags if needed local version_flags=" -X k8s.io/client-go/pkg/version.gitVersion=${GIT_VERSION} -X k8s.io/client-go/pkg/version.gitCommit=${GIT_COMMIT} @@ -232,6 +235,7 @@ function test() { octopus::log::info "running unit tests for adaptor ${adaptor}..." + # TODO adjust the test targets if needed local unit_test_targets=( "${CURR_DIR}/api/..." "${CURR_DIR}/cmd/..." @@ -266,7 +270,16 @@ function verify() { octopus::log::info "running integration tests for adaptor ${adaptor}..." + # TODO to implement the logic if needed, and place all integration tests to test/integration directory + # you can use the ginkgo cli: octopus::ginkgo::test "${CURR_DIR}/test/integration" + # or overwrite the preconfigure arguments: + # octopus::ginkgo::test -tags="test kubernetes-test" "${CURR_DIR}/test/integration" + # or use the raw `go test` with ginkgo: + # CGO_ENABLED=0 go test \ + # -tags=test \ + # "${CURR_DIR}/test/integration/..." -v -timeout=5m -tags=test -ginkgo.failFast -ginkgo.slowSpecThreshold=60 + # or use without ginkgo. octopus::log::info "...done" } @@ -277,7 +290,16 @@ function e2e() { octopus::log::info "running E2E tests for adaptor ${adaptor}..." + # TODO to implement the logic if needed, and place all E2E tests to test/e2e directory + # you can use the ginkgo cli: octopus::ginkgo::test "${CURR_DIR}/test/e2e" + # or overwrite the preconfigure arguments: + # octopus::ginkgo::test -tags="test kubernetes-test" "${CURR_DIR}/test/e2e" + # or use the raw `go test` with ginkgo: + # CGO_ENABLED=0 go test \ + # -tags=test \ + # "${CURR_DIR}/test/e2e/..." -v -timeout=5m -tags=test -ginkgo.failFast -ginkgo.slowSpecThreshold=60 + # or use without ginkgo. octopus::log::info "...done" } diff --git a/adaptors/mqtt/pkg/adaptor/service.go b/adaptors/mqtt/pkg/adaptor/service.go index b07f7c32..6beacfb1 100644 --- a/adaptors/mqtt/pkg/adaptor/service.go +++ b/adaptors/mqtt/pkg/adaptor/service.go @@ -1,13 +1,11 @@ package adaptor import ( - jsoniter "github.com/json-iterator/go" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" k8sruntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "github.com/rancher/octopus/adaptors/mqtt/api/v1alpha1" @@ -15,11 +13,16 @@ import ( api "github.com/rancher/octopus/pkg/adaptor/api/v1alpha1" "github.com/rancher/octopus/pkg/adaptor/connection" "github.com/rancher/octopus/pkg/adaptor/log" + "github.com/rancher/octopus/pkg/mqtt" + "github.com/rancher/octopus/pkg/util/converter" "github.com/rancher/octopus/pkg/util/object" ) func NewService() *Service { + mqtt.SetLogger(log.GetLogger()) + var scheme = k8sruntime.NewScheme() + // register v1alpha1 scheme into runtime scheme. utilruntime.Must(v1alpha1.AddToScheme(scheme)) return &Service{ @@ -41,76 +44,75 @@ func (s *Service) toJSON(in metav1.Object) []byte { } func (s *Service) Connect(server api.Connection_ConnectServer) error { - var device physical.Device + var holder physical.Device defer func() { - if device != nil { - device.Shutdown() + if holder != nil { + holder.Shutdown() } }() for { - var req, err = server.Recv() if err != nil { if !connection.IsClosed(err) { log.Error(err, "Failed to receive connect request from Limb") - return status.Errorf(codes.Unknown, "shutdown connection as receiving error from Limb") + return status.Error(codes.Unknown, "shutdown connection as receiving error from Limb") } return nil } - // validate device - var mqtt v1alpha1.MqttDevice - if err := jsoniter.Unmarshal(req.GetDevice(), &mqtt); err != nil { - return status.Errorf(codes.InvalidArgument, "failed to unmarshal device: %v", err) + // validates model GVK + var model = req.GetModel() + if model == nil { + return status.Error(codes.InvalidArgument, "invalid empty model") + } + var modelGVK = model.GroupVersionKind() + if modelGVK.Group != "devices.edge.cattle.io" { + return status.Errorf(codes.InvalidArgument, "invalid model group: %s", modelGVK.Group) } - if device == nil { - var deviceName = object.GetNamespacedName(&mqtt) - if deviceName.Namespace == "" || deviceName.Name == "" { - return status.Error(codes.InvalidArgument, "failed to recognize the empty device as the namespace/name is blank") + // processes device + switch modelGVK.Kind { + case "MQTTDevice": + // gets device spec + var device v1alpha1.MQTTDevice + if err := converter.UnmarshalJSON(req.GetDevice(), &device); err != nil { + return status.Errorf(codes.InvalidArgument, "failed to unmarshal device: %v", err) } - var dataHandler = func(name types.NamespacedName, status v1alpha1.MqttDeviceStatus) { - // send device by {name, namespace, status} tuple - var resp v1alpha1.MqttDevice - resp.Namespace = name.Namespace - resp.Name = name.Name - resp.Status = status + // creates device handler + if holder == nil { + // gets device namespaced name + var deviceName = object.GetNamespacedName(&device) + if deviceName.Namespace == "" || deviceName.Name == "" { + return status.Error(codes.InvalidArgument, "failed to recognize the empty device as the namespace/name is blank") + } - // convert device to json bytes - var respBytes = s.toJSON(&resp) + // gets log + var logger = log.WithValues("mqtt device", deviceName) - log.Info("dataHandler device update", "MqttDevice", string(respBytes)) + // creates handler for syncing to limb + var toLimb = func(in *v1alpha1.MQTTDevice) { + // convert device to json bytes + var respBytes = s.toJSON(in) - // send device - if err := server.Send(&api.ConnectResponse{Device: respBytes}); err != nil { - if !connection.IsClosed(err) { - log.Error(err, "Failed to send response to connection") + // send device to limb + if err := server.Send(&api.ConnectResponse{Device: respBytes}); err != nil { + if !connection.IsClosed(err) { + logger.Error(err, "Failed to send response to connection") + } } } - } - mqttClient, err := physical.NewMqttClient(mqtt.Name, mqtt.Spec.Config) - if err != nil { - log.Error(err, "connect receive new device NewMqttClient error") - return status.Errorf(codes.InvalidArgument, "failed to connect mqtt: %v", err) + holder = physical.NewDevice(logger, device.ObjectMeta, toLimb) } - device = physical.NewDevice( - log.WithValues("device", deviceName), - &mqtt, - dataHandler, - mqttClient, - ) - - go device.On() - - log.Info("connect receive new device success", "name", mqtt.Name) - - } else { - device.Configure(&mqtt.Spec) + // configures device + if err := holder.Configure(req.GetReferencesHandler(), &device); err != nil { + return status.Errorf(codes.InvalidArgument, "failed to configure the device: %v", err) + } + default: + return status.Errorf(codes.InvalidArgument, "invalid model kind: %s", modelGVK.Kind) } - } } diff --git a/adaptors/mqtt/pkg/physical/converter.go b/adaptors/mqtt/pkg/physical/converter.go deleted file mode 100644 index 391ca000..00000000 --- a/adaptors/mqtt/pkg/physical/converter.go +++ /dev/null @@ -1,180 +0,0 @@ -package physical - -import ( - "math" - "reflect" - "time" - - "github.com/rancher/octopus/adaptors/mqtt/api/v1alpha1" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -const ( - minInt53 = -2251799813685248 - maxInt53 = 2251799813685247 -) -const diffMin = 0.000001 - -func ConvertToStatusProperty(payload []byte, property *v1alpha1.Property) (v1alpha1.StatusProperty, error) { - var statusProperty v1alpha1.StatusProperty - statusProperty.Description = property.Description - statusProperty.Name = property.Name - statusProperty.UpdatedAt = metav1.Time{Time: time.Now()} - - if property.SubInfo.PayloadType != v1alpha1.PayloadTypeJSON { - return statusProperty, nil - } - - result := gjson.GetBytes(payload, property.JSONPath) - valueProps, err := ConvertResultToValueProps(result) - if err != nil { - return statusProperty, err - } - statusProperty.Value = valueProps - - return statusProperty, nil -} - -func ConvertResultToValueProps(result gjson.Result) (v1alpha1.ValueProps, error) { - var valueProps v1alpha1.ValueProps - var err error - switch result.Type { - case gjson.Null: - return valueProps, nil - case gjson.False, gjson.True: - valueProps.ValueType = v1alpha1.ValueTypeBoolean - boolValue := result.Bool() - valueProps.BooleanValue = boolValue - case gjson.String: - valueProps.ValueType = v1alpha1.ValueTypeString - stringValue := result.String() - valueProps.StringValue = stringValue - case gjson.JSON: - if result.IsArray() { - valueProps.ValueType = v1alpha1.ValueTypeArray - resultArray := result.Array() - for _, r := range resultArray { - props, err := ConvertResultToValueProps(r) - if err != nil { - continue - } - var arrayProps v1alpha1.ValueArrayProps - arrayProps.ValueProps = props - valueProps.ArrayValue = append(valueProps.ArrayValue, arrayProps) - } - } else if result.IsObject() { - valueProps.ValueType = v1alpha1.ValueTypeObject - valueProps.ObjectValue = new(runtime.RawExtension) - valueProps.ObjectValue.Raw = []byte(result.String()) - } - case gjson.Number: - n := int64(result.Num) - if float64(n) == result.Num && n >= minInt53 && n <= maxInt53 { - valueProps.ValueType = v1alpha1.ValueTypeInt - valueProps.IntValue = n - } else { - valueProps.ValueType = v1alpha1.ValueTypeFloat - valueProps.FloatValue = new(v1alpha1.ValueFloat) - valueProps.FloatValue.F = result.Num - } - } - return valueProps, err -} - -func ConvertValueToJSONPayload(payload []byte, property *v1alpha1.Property) ([]byte, error) { - var value interface{} - var err error - switch property.Value.ValueType { - case v1alpha1.ValueTypeArray: - var valueSlice []interface{} - for _, i := range property.Value.ArrayValue { - v, err := ConvertValueArrayPropsToInterface(i) - if err != nil { - continue - } - valueSlice = append(valueSlice, v) - } - value = valueSlice - case v1alpha1.ValueTypeObject: - value = property.Value.ObjectValue - case v1alpha1.ValueTypeInt: - value = property.Value.IntValue - case v1alpha1.ValueTypeString: - value = property.Value.StringValue - case v1alpha1.ValueTypeBoolean: - value = property.Value.BooleanValue - case v1alpha1.ValueTypeFloat: - value = property.Value.FloatValue.F - } - - nValue, err := sjson.SetBytes(payload, property.JSONPath, value) - - if err != nil { - return nil, err - } - - return nValue, nil - -} - -func ConvertValueArrayPropsToInterface(props v1alpha1.ValueArrayProps) (interface{}, error) { - var value interface{} - switch props.ValueType { - case v1alpha1.ValueTypeArray: - var valueSlice []interface{} - for _, i := range props.ArrayValue { - v, err := ConvertValueArrayPropsToInterface(i) - if err != nil { - return value, err - } - valueSlice = append(valueSlice, v) - } - value = valueSlice - case v1alpha1.ValueTypeObject: - value = props.ObjectValue - case v1alpha1.ValueTypeInt: - value = props.IntValue - case v1alpha1.ValueTypeString: - value = props.StringValue - case v1alpha1.ValueTypeBoolean: - value = props.BooleanValue - case v1alpha1.ValueTypeFloat: - value = props.FloatValue.F - } - - return value, nil -} - -func ComparativeValueProps(l, r v1alpha1.ValueProps) bool { - switch l.ValueType { - case v1alpha1.ValueTypeArray: - return reflect.DeepEqual(l.ArrayValue, r.ArrayValue) - case v1alpha1.ValueTypeObject: - ls := string(l.ObjectValue.Raw) - rs := string(r.ObjectValue.Raw) - if ls != rs { - return false - } - case v1alpha1.ValueTypeInt: - if l.IntValue != r.IntValue { - return false - } - case v1alpha1.ValueTypeString: - if l.StringValue != r.StringValue { - return false - } - case v1alpha1.ValueTypeBoolean: - if l.BooleanValue != r.BooleanValue { - return false - } - case v1alpha1.ValueTypeFloat: - if math.Dim(l.FloatValue.F, r.FloatValue.F) < diffMin { - return false - } - } - - return true -} diff --git a/adaptors/mqtt/pkg/physical/converter_test.go b/adaptors/mqtt/pkg/physical/converter_test.go deleted file mode 100644 index 08764aac..00000000 --- a/adaptors/mqtt/pkg/physical/converter_test.go +++ /dev/null @@ -1,274 +0,0 @@ -package physical - -import ( - "log" - "reflect" - "testing" - "time" - - "github.com/rancher/octopus/adaptors/mqtt/api/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - k8sruntime "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" -) - -var simpleData = []byte(`{ - "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "red", - "price": 19.95 - } - } - }`) - -var roomLightData = []byte(`{"switch":"off","brightness"：4,"power":{"powerDissipation":"10KWH","electricQuantity":19.99}}`) - -func TestConvertToStatusProperty(t *testing.T) { - - tests := map[string]struct { - input []byte - jsonPath string - want []byte - }{ - "simple": { - input: simpleData, - jsonPath: "store.bicycle.price", - want: []byte(`{"apiVersion":"devices.edge.cattle.io/v1alpha1","kind":"MqttDevice","metadata":{"creationTimestamp":null,"name":"testDevice"},"spec":{"config":{"broker":""},"properties":[{"description":"test property","jsonPath":"store.bicycle.price","name":"test_property","pubInfo":{"qos":0,"topic":""},"subInfo":{"payloadType":"json","qos":2,"topic":"test/abc"},"value":{"valueType":""}}]},"status":{"properties":[{"description":"test property","name":"test_property","updateAt":"2020-01-01T01:01:01Z","value":{"floatValue":"19.950000","valueType":"float"}}]}}`), - }, - "room light": { - input: roomLightData, - jsonPath: "power", - want: []byte(`{"apiVersion":"devices.edge.cattle.io/v1alpha1","kind":"MqttDevice","metadata":{"creationTimestamp":null,"name":"testDevice"},"spec":{"config":{"broker":""},"properties":[{"description":"test property","jsonPath":"power","name":"test_property","pubInfo":{"qos":0,"topic":""},"subInfo":{"payloadType":"json","qos":2,"topic":"test/abc"},"value":{"valueType":""}}]},"status":{"properties":[{"description":"test property","name":"test_property","updateAt":"2020-01-01T01:01:01Z","value":{"objectValue":{"electricQuantity":19.99,"powerDissipation":"10KWH"},"valueType":"object"}}]}}`), - }, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - var property v1alpha1.Property - property.JSONPath = tc.jsonPath - property.Name = "test_property" - property.Description = "test property" - property.SubInfo.PayloadType = v1alpha1.PayloadTypeJSON - property.SubInfo.Topic = "test/abc" - property.SubInfo.Qos = 2 - statusProperty, err := ConvertToStatusProperty(tc.input, &property) - if err != nil { - t.Fatal("ConvertToStatusProperty error:", err) - return - } - tm := time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC) - statusProperty.UpdatedAt = metav1.Time{Time: tm} - var device v1alpha1.MqttDevice - device.APIVersion = "v1alpha1" - device.Kind = "MqttDevice" - device.Name = "testDevice" - device.Spec.Properties = append(device.Spec.Properties, property) - device.Status.Properties = append(device.Status.Properties, statusProperty) - - var out = unstructured.Unstructured{Object: make(map[string]interface{})} - var scheme = k8sruntime.NewScheme() - utilruntime.Must(v1alpha1.AddToScheme(scheme)) - scheme.Convert(&device, &out, nil) - var bytes, _ = out.MarshalJSON() - got := bytes[:len(bytes)-1] - - if !reflect.DeepEqual(got, tc.want) { - log.Fatalf("expected: %s , got: %s", string(tc.want), string(bytes)) - } - }) - } -} - -func TestConvertValueToJSONPayload(t *testing.T) { - - tests := map[string]struct { - input []byte - jsonPath string - want []byte - }{ - "simple string": { - input: simpleData, - jsonPath: "store.bicycle.color", - want: []byte(`{ - "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "huang", - "price": 19.95 - } - } - }`), - }, - "simple float": { - input: simpleData, - jsonPath: "store.bicycle.price", - want: []byte(`{ - "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "red", - "price": 20.11 - } - } - }`), - }, - "simple array": { - input: simpleData, - jsonPath: "store.book", - want: []byte(`{ - "store": { - "book": [10,30], - "bicycle": { - "color": "red", - "price": 19.95 - } - } - }`), - }, - "simple object": { - input: simpleData, - jsonPath: "store.bicycle", - want: []byte(`{ - "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": {"color":"black","price":222.77} - } - }`), - }, - } - - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - var property v1alpha1.Property - property.JSONPath = tc.jsonPath - property.Name = "test_property" - property.Description = "test property" - property.SubInfo.PayloadType = v1alpha1.PayloadTypeJSON - property.SubInfo.Topic = "test/abc" - property.SubInfo.Qos = 2 - - switch name { - case "simple string": - property.Value.ValueType = v1alpha1.ValueTypeString - property.Value.StringValue = "huang" - case "simple float": - property.Value.ValueType = v1alpha1.ValueTypeFloat - property.Value.FloatValue = new(v1alpha1.ValueFloat) - property.Value.FloatValue.F = float64(20.11) - case "simple array": - property.Value.ValueType = v1alpha1.ValueTypeArray - item1 := v1alpha1.ValueArrayProps{ValueProps: v1alpha1.ValueProps{ValueType: v1alpha1.ValueTypeInt, IntValue: 10}} - item2 := v1alpha1.ValueArrayProps{ValueProps: v1alpha1.ValueProps{ValueType: v1alpha1.ValueTypeInt, IntValue: 30}} - property.Value.ArrayValue = append(property.Value.ArrayValue, item1, item2) - case "simple object": - property.Value.ValueType = v1alpha1.ValueTypeObject - property.Value.ObjectValue = new(k8sruntime.RawExtension) - property.Value.ObjectValue.Raw = []byte(`{"color":"black","price":222.77}`) - } - - newPayload, err := ConvertValueToJSONPayload(simpleData, &property) - if err != nil { - t.Fatal(err) - return - } - - if !reflect.DeepEqual(newPayload, tc.want) { - log.Fatalf("expected: %s , got: %s", string(tc.want), string(newPayload)) - } - }) - } -} diff --git a/adaptors/mqtt/pkg/physical/device.go b/adaptors/mqtt/pkg/physical/device.go index 6386bf61..f59ce7af 100644 --- a/adaptors/mqtt/pkg/physical/device.go +++ b/adaptors/mqtt/pkg/physical/device.go @@ -1,305 +1,307 @@ package physical import ( - "errors" + "reflect" "sync" - "time" - MQTT "github.com/eclipse/paho.mqtt.golang" "github.com/go-logr/logr" + "github.com/pkg/errors" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/rancher/octopus/adaptors/mqtt/api/v1alpha1" + api "github.com/rancher/octopus/pkg/adaptor/api/v1alpha1" + "github.com/rancher/octopus/pkg/mqtt" + "github.com/rancher/octopus/pkg/util/converter" "github.com/rancher/octopus/pkg/util/object" - "k8s.io/apimachinery/pkg/util/sets" -) - -const ( - disconnectQuiesce = 1000 - waitTimeout = time.Second * 10 -) - -var ( - errorBrokerConnectTimeout = errors.New("Broker Connect Timeout") - errorSubscribeTimeout = errors.New("Subscribe Timeout") - errorUnsubscribeTimeout = errors.New("Unsubscribe Timeout") - errorPublishTimeout = errors.New("Publish Timeout") ) +// Device is an interface for device operations set. type Device interface { - Configure(spec *v1alpha1.MqttDeviceSpec) - On() + // Shutdown uses to close the connection between adaptor and real(physical) device. Shutdown() + // Configure uses to set up the device. + Configure(references api.ReferencesHandler, configuration interface{}) error } -func NewDevice(log logr.Logger, obj *v1alpha1.MqttDevice, handler DataHandler, client MQTT.Client) Device { - d := device{ - client: client, - handler: handler, - stop: make(chan struct{}), - log: log, +// NewDevice creates a Device. +func NewDevice(log logr.Logger, meta metav1.ObjectMeta, toLimb MQTTDeviceLimbSyncer) Device { + log.Info("Created") + return &mqttDevice{ + log: log, + instance: &v1alpha1.MQTTDevice{ + ObjectMeta: meta, + }, + toLimb: toLimb, } - obj.DeepCopyInto(&d.obj) - - return &d } -type device struct { +type mqttDevice struct { sync.Mutex - log logr.Logger - client MQTT.Client - handler DataHandler - obj v1alpha1.MqttDevice - stop chan struct{} - payloadMap sync.Map - o sync.Once -} -func (dev *device) On() { - if err := dev.subscribe(); err != nil { - dev.log.Error(err, "device subscribe error") - close(dev.stop) - return - } + log logr.Logger - select { - case <-dev.stop: - dev.log.Info("device is stop") - return - } + instance *v1alpha1.MQTTDevice + toLimb MQTTDeviceLimbSyncer + mqttClient mqtt.Client } -func (dev *device) Configure(spec *v1alpha1.MqttDeviceSpec) { - dev.Lock() - defer dev.Unlock() - if err := dev.updateSubscription(spec); err != nil { - dev.log.Error(err, "device Configure updateSubscription error") - return +func (d *mqttDevice) Shutdown() { + d.log.Info("Shutdown") + if d.mqttClient != nil { + d.mqttClient.Disconnect() + d.mqttClient = nil + d.log.V(1).Info("Disconnected connection") } - if err := dev.publishProperties(spec.Properties); err != nil { - dev.log.Error(err, "device Configure publish error") - return - } - dev.removeRedundantStatus(spec) - dev.updateDeviceSpec(spec) } -func (dev *device) Shutdown() { - dev.o.Do(func() { - close(dev.stop) - dev.unsubscribeAll() - dev.client.Disconnect(disconnectQuiesce) - }) -} +func (d *mqttDevice) Configure(references api.ReferencesHandler, configuration interface{}) error { + var device, ok = configuration.(*v1alpha1.MQTTDevice) + if !ok { + d.log.Error(errors.New("invalidate configuration type"), "Failed to configure") + return nil + } + var newSpec = device.Spec -func (dev *device) publishProperties(properties []v1alpha1.Property) error { - for _, property := range properties { - if property.SubInfo.PayloadType != v1alpha1.PayloadTypeJSON { - continue - } - var statusProperty v1alpha1.StatusProperty - for _, sp := range dev.obj.Status.Properties { - if sp.Name == property.Name { - statusProperty = sp - break - } - } - if ComparativeValueProps(property.Value, statusProperty.Value) { - continue - } + d.Lock() + defer d.Unlock() - ivalue, ok := dev.payloadMap.Load(property.SubInfo.Topic) - if !ok { - continue + if !reflect.DeepEqual(d.instance.Spec.Protocol, newSpec.Protocol) { + if d.mqttClient != nil { + d.mqttClient.Disconnect() + d.mqttClient = nil + d.log.V(1).Info("Disconnected stale connection") } - payload := ivalue.([]byte) - newValuePayload, err := ConvertValueToJSONPayload(payload, &property) + + var cli, err = mqtt.NewClient(newSpec.Protocol.MQTTOptions, object.GetControlledOwnerObjectReference(device), references) if err != nil { - return err + return errors.Wrap(err, "failed to create MQTT client") } - var pubTopic string - var qos byte - if property.PubInfo.Topic == "" { - pubTopic = property.SubInfo.Topic - qos = byte(property.SubInfo.Qos) - } else { - pubTopic = property.PubInfo.Topic - qos = byte(property.PubInfo.Qos) + err = cli.Connect() + if err != nil { + return errors.Wrap(err, "failed to connect MQTT broker") } + d.mqttClient = cli + d.log.V(1).Info("Connected to MQTT broker") + } - dev.log.Info("device publish cmd", "payload", string(newValuePayload), "propertyName", property.Name, "pubTopic", pubTopic) + return d.refresh(newSpec) +} - token := dev.client.Publish(pubTopic, qos, true, newValuePayload) - // TODO change to WaitTimeout , but func have bug in lib - if token.Wait() && token.Error() != nil { - return err +func (d *mqttDevice) refresh(newSpec v1alpha1.MQTTDeviceSpec) error { + // indexes stale status properties + var staleStatusPropsIndex = make(map[string]v1alpha1.MQTTDeviceStatusProperty, len(d.instance.Status.Properties)) + for _, prop := range d.instance.Status.Properties { + staleStatusPropsIndex[prop.Name] = prop + } + + // constructs status properties + var newStatusProps = make([]v1alpha1.MQTTDeviceStatusProperty, 0, len(newSpec.Properties)) + for _, newSpecProp := range newSpec.Properties { + switch newSpec.Protocol.Pattern { + case v1alpha1.MQTTDevicePatternAttributedMessage: + if newSpecProp.ReadOnly != nil && !*newSpecProp.ReadOnly { + if err := verifyWritableJSONPath(getPath(newSpecProp.Name, newSpecProp.Path)); err != nil { + return errors.Wrapf(err, "illegal path %s", getPath(newSpecProp.Name, newSpecProp.Path)) + } + } } - dev.payloadMap.Store(property.SubInfo.Topic, newValuePayload) + var newStatusProp = v1alpha1.MQTTDeviceStatusProperty{ + MQTTDeviceProperty: newSpecProp, + } + var staleStatusProp, exist = staleStatusPropsIndex[newSpecProp.Name] + if !exist { + newStatusProp.Value = nil + } else { + newStatusProp.Value = staleStatusProp.Value + newStatusProp.UpdatedAt = staleStatusProp.UpdatedAt + } + newStatusProps = append(newStatusProps, newStatusProp) } - return nil -} -func (dev *device) subscribe() error { - filters := make(map[string]byte, len(dev.obj.Spec.Properties)) - for _, property := range dev.obj.Spec.Properties { - filters[property.SubInfo.Topic] = byte(property.SubInfo.Qos) + // indexes stale spec properties + var staleSpecPropsIndex = make(map[string]v1alpha1.MQTTDeviceProperty, len(d.instance.Spec.Properties)) + for _, prop := range d.instance.Spec.Properties { + staleSpecPropsIndex[prop.Name] = prop } - token := dev.client.SubscribeMultiple(filters, dev.callback) - // TODO change to WaitTimeout , but func have bug in lib - if token.Wait() && token.Error() != nil { - return token.Error() + // refreshes + switch newSpec.Protocol.Pattern { + case v1alpha1.MQTTDevicePatternAttributedMessage: + if err := d.refreshAsAttributedMessage(staleSpecPropsIndex, newSpec.Properties); err != nil { + return err + } + case v1alpha1.MQTTDevicePatternAttributeTopic: + if err := d.refreshAsAttributedTopic(staleSpecPropsIndex, newSpec.Properties); err != nil { + return err + } + default: + return errors.Errorf("failed to recognize protocol pattern %s", newSpec.Protocol.Pattern) } + // records + d.instance.Spec = newSpec + d.instance.Status = v1alpha1.MQTTDeviceStatus{Properties: newStatusProps} + d.toLimb(d.instance) return nil } -func (dev *device) callback(client MQTT.Client, msg MQTT.Message) { - dev.Lock() - defer dev.Unlock() - dev.log.Info("device subscribe callback", "msg", string(msg.Payload()), "topic", msg.Topic()) - dev.log.Info("device current object", "object", dev.obj) - dev.payloadMap.Store(msg.Topic(), msg.Payload()) - for _, property := range dev.obj.Spec.Properties { - if property.SubInfo.Topic == msg.Topic() { - statusProperty, err := ConvertToStatusProperty(msg.Payload(), &property) - if err != nil { - dev.log.Error(err, "device subscribe callback ConvertToStatusProperty error", "property", property) - continue +func (d *mqttDevice) refreshAsAttributedMessage(staleSpecPropsIndex map[string]v1alpha1.MQTTDeviceProperty, newSpecProps []v1alpha1.MQTTDeviceProperty) error { + // subscribes + var subscribeTopics = []mqtt.SubscribeTopic{{}} + var subscribeHandler = func(msg mqtt.SubscribeMessage) { + // receives and updates status properties + d.Lock() + defer d.Unlock() + + var payload = msg.Payload + for idx, prop := range d.instance.Status.Properties { + var propValue = &v1alpha1.MQTTDevicePropertyValue{} + var result = gjson.GetBytes(payload, getPath(prop.Name, prop.Path)) + if result.Index > 0 { + propValue.Raw = payload[result.Index : result.Index+len(result.Raw)] + } else { + propValue.Raw = []byte(result.Raw) } + prop.Value = propValue + prop.UpdatedAt = now() + d.instance.Status.Properties[idx] = prop + } + d.toLimb(d.instance) + } + if err := d.mqttClient.Subscribe(subscribeTopics, subscribeHandler); err != nil { + return errors.Wrap(err, "failed to subscribe") + } - dev.log.Info("device subscribe callback ConvertToStatusProperty", "statusProperty", statusProperty) - - var found bool - for j, curStatusProperty := range dev.obj.Status.Properties { - if curStatusProperty.Name == property.Name { - statusProperty.DeepCopyInto(&dev.obj.Status.Properties[j]) - found = true + // publishes + var stalePayload []byte + var payload []byte + for _, newSpecProp := range newSpecProps { + if newSpecProp.ReadOnly != nil && !*newSpecProp.ReadOnly { + // constructs stale payload + if staleSpecProp, exist := staleSpecPropsIndex[newSpecProp.Name]; exist { + if staleSpecProp.Value != nil { + var stalePropPath = getPath(staleSpecProp.Name, staleSpecProp.Path) + stalePayload, _ = sjson.SetBytes(payload, stalePropPath, staleSpecProp.Value) } } - if !found { - dev.obj.Status.Properties = append(dev.obj.Status.Properties, statusProperty) + + // constructs new payload + if newSpecProp.Value != nil { + var newPropPath = getPath(newSpecProp.Name, newSpecProp.Path) + var err error + payload, err = sjson.SetBytes(payload, newPropPath, newSpecProp.Value) + if err != nil { + return errors.Wrapf(err, "failed to set property value on path: %s", newPropPath) + } } } } - - dev.handler(object.GetNamespacedName(&dev.obj), dev.obj.Status) -} - -func (dev *device) updateSubscription(spec *v1alpha1.MqttDeviceSpec) error { - if err := dev.unsubscribeOld(spec); err != nil { - return err - } - if err := dev.reSubscribeAll(spec); err != nil { - return err + if !reflect.DeepEqual(stalePayload, payload) { + if err := d.mqttClient.Publish(mqtt.PublishMessage{Payload: payload}); err != nil { + return errors.Wrap(err, "failed to publish") + } } return nil } -func (dev *device) unsubscribeOld(spec *v1alpha1.MqttDeviceSpec) error { - oldTopicSet := sets.NewString() - for _, property := range dev.obj.Spec.Properties { - oldTopicSet.Insert(property.SubInfo.Topic) - } - - newTopicSet := sets.NewString() - for _, property := range spec.Properties { - newTopicSet.Insert(property.SubInfo.Topic) +func (d *mqttDevice) refreshAsAttributedTopic(staleSpecPropsIndex map[string]v1alpha1.MQTTDeviceProperty, newSpecProps []v1alpha1.MQTTDeviceProperty) error { + // subscribes spec properties + var subscribeTopics = make([]mqtt.SubscribeTopic, 0, len(newSpecProps)) + for idx, newSpecProp := range newSpecProps { + // appends subscribe topic + subscribeTopics = append(subscribeTopics, mqtt.SubscribeTopic{ + Index: idx, + Render: getSubscribeRender(&newSpecProp), + QoSPointer: (*byte)(newSpecProp.QoS), + }) } + var subscribeHandler = func(msg mqtt.SubscribeMessage) { + // receives and updates status properties + d.Lock() + defer d.Unlock() - allTopicSet := oldTopicSet.Union(newTopicSet) - mustDelTopicSet := allTopicSet.Difference(newTopicSet) + if msg.Index > len(d.instance.Status.Properties) { + return + } - var unSubTopic []string - for _, topic := range mustDelTopicSet.List() { - unSubTopic = append(unSubTopic, topic) + var propValue v1alpha1.MQTTDevicePropertyValue + if err := converter.UnmarshalJSON(msg.Payload, &propValue); err != nil { + d.log.Error(err, "Failed to unmarshal subscribed payload", "topic", msg.Topic) + return + } + var prop = &d.instance.Status.Properties[msg.Index] + prop.Value = &propValue + prop.UpdatedAt = now() + // TODO should we debounce here? + d.toLimb(d.instance) + } + if err := d.mqttClient.Subscribe(subscribeTopics, subscribeHandler); err != nil { + return errors.Wrap(err, "failed to subscribe") } - if len(unSubTopic) > 1 { - token := dev.client.Unsubscribe(unSubTopic...) - // TODO change to WaitTimeout , but func have bug in lib - if token.Wait() && token.Error() != nil { - return token.Error() + // publishes writable spec properties + for _, newSpecProp := range newSpecProps { + if newSpecProp.ReadOnly != nil && !*newSpecProp.ReadOnly { + var staleSpecProp = staleSpecPropsIndex[newSpecProp.Name] + // publishes again if changed + if !reflect.DeepEqual(staleSpecProp, newSpecProp) { + var err = d.mqttClient.Publish(mqtt.PublishMessage{ + Render: getPublishRender(&newSpecProp), + QoSPointer: (*byte)(newSpecProp.QoS), + RetainedPointer: newSpecProp.Retained, + Payload: newSpecProp.Value, + }) + if err != nil { + return errors.Wrapf(err, "failed to publish property %s", newSpecProp.Name) + } + } } } return nil } -func (dev *device) reSubscribeAll(spec *v1alpha1.MqttDeviceSpec) error { - dev.client.Disconnect(disconnectQuiesce) - - var err error - if dev.client, err = NewMqttClient(dev.obj.Name, spec.Config); err != nil { - return err +func getPath(name, path string) string { + if path != "" { + return path } + return name +} - filters := make(map[string]byte, len(spec.Properties)) - for _, property := range spec.Properties { - filters[property.SubInfo.Topic] = byte(property.SubInfo.Qos) - } +func getPublishRender(prop *v1alpha1.MQTTDeviceProperty) map[string]string { + var render = make(map[string]string, 2) + + // gets path rendering value + render["path"] = getPath(prop.Name, prop.Path) - token := dev.client.SubscribeMultiple(filters, dev.callback) - // TODO change to WaitTimeout , but func have bug in lib - if token.Wait() && token.Error() != nil { - return token.Error() + // gets operator rendering value + if prop.Operator != nil { + render["operator"] = prop.Operator.Write } - return nil + return render } -func (dev *device) removeRedundantStatus(spec *v1alpha1.MqttDeviceSpec) { - for i := 0; i < len(dev.obj.Status.Properties); i++ { - statusProperty := dev.obj.Status.Properties[i] - var found bool - for _, property := range spec.Properties { - if property.Name == statusProperty.Name { - found = true - break - } - } - if !found { - dev.obj.Status.Properties = append(dev.obj.Status.Properties[:i], dev.obj.Status.Properties[i+1:]...) - } - } -} +func getSubscribeRender(prop *v1alpha1.MQTTDeviceProperty) map[string]string { + var render = make(map[string]string, 2) -func (dev *device) unsubscribeAll() { - subSet := sets.NewString() - for _, property := range dev.obj.Spec.Properties { - if subSet.HasAny(property.SubInfo.Topic) { - continue - } - subSet.Insert(property.SubInfo.Topic) - } - token := dev.client.Unsubscribe(subSet.List()...) - // TODO change to WaitTimeout , but func have bug in lib - if token.Wait() && token.Error() != nil { - dev.log.Error(token.Error(), "device unsubscribeAll error") + // gets path rendering value + render["path"] = getPath(prop.Name, prop.Path) + + // gets operator rendering value + if prop.Operator != nil { + render["operator"] = prop.Operator.Read } - return -} -func (dev *device) updateDeviceSpec(spec *v1alpha1.MqttDeviceSpec) { - spec.DeepCopyInto(&dev.obj.Spec) + return render } -func NewMqttClient(clientID string, config v1alpha1.MqttConfig) (MQTT.Client, error) { - opts := MQTT.NewClientOptions() - opts.AddBroker(config.Broker) - opts.SetClientID(clientID) - opts.SetUsername(config.Username) - opts.SetPassword(config.Password) - opts.SetOrderMatters(true) - opts.SetAutoReconnect(true) - opts.SetCleanSession(false) - - client := MQTT.NewClient(opts) - token := client.Connect() - if token.Wait() && token.Error() != nil { - return nil, token.Error() - } - - return client, nil +func now() *metav1.Time { + var ret = metav1.Now() + return &ret } diff --git a/adaptors/mqtt/pkg/physical/handler.go b/adaptors/mqtt/pkg/physical/handler.go index a68dc27b..c1ad615a 100644 --- a/adaptors/mqtt/pkg/physical/handler.go +++ b/adaptors/mqtt/pkg/physical/handler.go @@ -1,9 +1,8 @@ package physical import ( - "k8s.io/apimachinery/pkg/types" - "github.com/rancher/octopus/adaptors/mqtt/api/v1alpha1" ) -type DataHandler func(name types.NamespacedName, status v1alpha1.MqttDeviceStatus) +// MQTTDeviceLimbSyncer is used to sync mqtt device to limb. +type MQTTDeviceLimbSyncer func(in *v1alpha1.MQTTDevice) diff --git a/adaptors/mqtt/pkg/physical/parameters.go b/adaptors/mqtt/pkg/physical/parameters.go deleted file mode 100644 index 5ccb9a19..00000000 --- a/adaptors/mqtt/pkg/physical/parameters.go +++ /dev/null @@ -1,28 +0,0 @@ -package physical - -import ( - "time" -) - -const ( - defaultSyncInterval = 5 - defaultTimeout = 30 -) - -type Parameters struct { - SyncInterval time.Duration `json:"syncInterval,omitempty"` - Timeout time.Duration `json:"timeout,omitempty"` -} - -func (p *Parameters) Validate() error { - // nothing to do - - return nil -} - -func DefaultParameters() Parameters { - return Parameters{ - SyncInterval: defaultSyncInterval, - Timeout: defaultTimeout, - } -} diff --git a/adaptors/mqtt/pkg/physical/validation.go b/adaptors/mqtt/pkg/physical/validation.go new file mode 100644 index 00000000..db19f748 --- /dev/null +++ b/adaptors/mqtt/pkg/physical/validation.go @@ -0,0 +1,40 @@ +package physical + +import ( + "github.com/pkg/errors" +) + +// verifyWritableJSONPath verifies the legality of path which is used for writing value, +// this inspires by https://github.com/tidwall/sjson#path-syntax, in order to further ensure the ability to write back, +// the following checks were made: +// - `children.-1` is not allowed; +// - `children|@reverse` is not allowed; +// - `child*.2` is not allowed; +// - `c?ildren.0` is not allowed; +// - `friends.#.first` is not allowed. +func verifyWritableJSONPath(path string) error { + for i := 0; i < len(path); i++ { + var p = path[i] + switch p { + case '\\': + i++ + if i < len(path) { + i++ + } + case '.': + var next = path[i+1:] + if len(next) > 1 && next[0] == '-' { + return errors.New("minus character not allowed in path") + } + case '*', '?': + return errors.New("wildcard characters not allowed in path") + case '#': + return errors.New("array access character not allowed in path") + case '@': + return errors.New("modifiers not allowed in path") + case '|': + return errors.New("pipe characters not allowed in path") + } + } + return nil +} diff --git a/adaptors/mqtt/pkg/physical/validation_test.go b/adaptors/mqtt/pkg/physical/validation_test.go new file mode 100644 index 00000000..9cc86b0d --- /dev/null +++ b/adaptors/mqtt/pkg/physical/validation_test.go @@ -0,0 +1,63 @@ +package physical + +import ( + "fmt" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func Test_verifyWritableJSONPath(t *testing.T) { + var testCases = []struct { + name string + given string + expected error + }{ + { + name: "escape blank", + given: "a\\ b.c", + expected: nil, + }, + { + name: "escape blank with minus", + given: "a\\ -b.c", + expected: nil, + }, + { + name: "std", + given: "a.b.c", + expected: nil, + }, + { + name: "with minus", + given: "a.b.-1", + expected: errors.New("minus character not allowed in path"), + }, + { + name: "with wildcard", + given: "child*.2", + expected: errors.New("wildcard characters not allowed in path"), + }, + { + name: "with wildcard", + given: "c?ildren.0", + expected: errors.New("wildcard characters not allowed in path"), + }, + { + name: "with modifier", + given: "@pretty:{\"sortKeys\":true}", + expected: errors.New("modifiers not allowed in path"), + }, + { + name: "with pipe", + given: "a.b|@reverse", + expected: errors.New("pipe characters not allowed in path"), + }, + } + + for _, tc := range testCases { + var actual = verifyWritableJSONPath(tc.given) + assert.Equal(t, fmt.Sprint(tc.expected), fmt.Sprint(actual), "case %q", tc.name) + } +} diff --git a/adaptors/mqtt/test/e2e/.gitkeep b/adaptors/mqtt/test/e2e/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/adaptors/mqtt/test/integration/.gitkeep b/adaptors/mqtt/test/integration/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/adaptors/mqtt/test/integration/adaptor/connection_test.go b/adaptors/mqtt/test/integration/adaptor/connection_test.go deleted file mode 100644 index fede5f7b..00000000 --- a/adaptors/mqtt/test/integration/adaptor/connection_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package adaptor - -import ( - "io" - - "github.com/golang/mock/gomock" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - grpccodes "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/rancher/octopus/adaptors/mqtt/pkg/adaptor" - "github.com/rancher/octopus/pkg/adaptor/api/v1alpha1" - mock_v1alpha1 "github.com/rancher/octopus/pkg/adaptor/api/v1alpha1/mock" -) - -// testing scenarios: -// + Server -// - validate if the connection stop when it closes -// - validate the process of input parameters -// - validate the process of input device -var _ = Describe("Connection", func() { - var ( - err error - - mockCtrl *gomock.Controller - service *adaptor.Service - ) - - BeforeEach(func() { - mockCtrl = gomock.NewController(GinkgoT()) - service = adaptor.NewService() - }) - - AfterEach(func() { - mockCtrl.Finish() - }) - - Context("Server", func() { - - var mockServer *mock_v1alpha1.MockConnection_ConnectServer - - BeforeEach(func() { - mockServer = mock_v1alpha1.NewMockConnection_ConnectServer(mockCtrl) - }) - - It("should be stopped if closed", func() { - // io.EOF - mockServer.EXPECT().Recv().Return(nil, io.EOF) - err = service.Connect(mockServer) - Expect(err).ToNot(HaveOccurred()) - - // canceled by context - mockServer.EXPECT().Recv().Return(nil, status.Error(grpccodes.Canceled, "context canceled")) - err = service.Connect(mockServer) - Expect(err).ToNot(HaveOccurred()) - - // other canceled reason - mockServer.EXPECT().Recv().Return(nil, status.Error(grpccodes.Canceled, "other")) - err = service.Connect(mockServer) - Expect(err).To(HaveOccurred()) - - // transport is closing - mockServer.EXPECT().Recv().Return(nil, status.Error(grpccodes.Unavailable, "transport is closing")) - err = service.Connect(mockServer) - Expect(err).ToNot(HaveOccurred()) - - // other unavailable reason - mockServer.EXPECT().Recv().Return(nil, status.Error(grpccodes.Unavailable, "other")) - err = service.Connect(mockServer) - Expect(err).To(HaveOccurred()) - }) - - It("should process the input device", func() { - // failed unmarshal - mockServer.EXPECT().Recv().Return(&v1alpha1.ConnectRequest{ - Model: &metav1.TypeMeta{ - APIVersion: "devices.edge.cattle.io/v1alpha1", - Kind: "MqttDevice", - }, - Device: []byte(`{this is an illegal json}`), - }, nil) - err = service.Connect(mockServer) - var sts = status.Convert(err) - Expect(sts.Code()).To(Equal(grpccodes.InvalidArgument)) - Expect(sts.Message()).To(HavePrefix("failed to unmarshal device")) - - // failed to connect a device - mockServer.EXPECT().Recv().Return(&v1alpha1.ConnectRequest{ - Model: &metav1.TypeMeta{ - APIVersion: "devices.edge.cattle.io/v1alpha1", - Kind: "MqttDevice", - }, - Device: []byte(` - { - "apiVersion": "devices.edge.cattle.io/v1alpha1", - "kind": "MqttDevice", - "metadata": { - "name": "testDevice", - "namespace": "default" - }, - "spec": { - "config": { - "broker": "tcp://127.0.0.1:1883", - "password": "parchk", - "username": "test123" - }, - "properties": [{ - "description": "test property", - "jsonPath": "switch", - "name": "test_property", - "subInfo": { - "payloadType": "json", - "qos": 2, - "topic": "/device/room/light" - } - }] - } - }`), - }, nil) - err = service.Connect(mockServer) - sts = status.Convert(err) - Expect(sts.Code()).To(Equal(grpccodes.InvalidArgument)) - Expect(sts.Message()).To(Equal("failed to connect mqtt: Network Error : dial tcp 127.0.0.1:1883: connect: connection refused")) - }) - - }) - -}) diff --git a/adaptors/mqtt/test/integration/adaptor/suite_test.go b/adaptors/mqtt/test/integration/adaptor/suite_test.go deleted file mode 100644 index e919ebec..00000000 --- a/adaptors/mqtt/test/integration/adaptor/suite_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package adaptor - -import ( - "context" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" -) - -var ( - testCtx context.Context - testCtxCancel context.CancelFunc -) - -func TestAdaptor(t *testing.T) { - defer GinkgoRecover() - - RegisterFailHandler(Fail) - - RunSpecsWithDefaultAndCustomReporters(t, - "adaptor suite", - []Reporter{printer.NewlineReporter{}}) -} - -var _ = BeforeSuite(func(done Done) { - testCtx, testCtxCancel = context.WithCancel(context.Background()) - - close(done) -}, 600) - -var _ = AfterSuite(func() { - if testCtxCancel != nil { - testCtxCancel() - } -}, 600) diff --git a/adaptors/mqtt/test/quickstart.md b/adaptors/mqtt/test/quickstart.md deleted file mode 100644 index ca97c558..00000000 --- a/adaptors/mqtt/test/quickstart.md +++ /dev/null @@ -1,97 +0,0 @@ -#Quick Start - -1.Install and run k3s (Can use [k3d](https://github.com/rancher/k3d) to get your cluster up and running quickly) -```shell script -k3d create -``` -2.deploy octopus in your k3s cluster use [all_in_one.yaml](../../../../deploy/e2e) -```shell script -kubectl apply -f all_in_one.yaml -``` -3.deploy MQTT adaptor use [all_in_one.yaml](../../deploy/e2e) -```shell script -kubectl apply -f all_in_one.yaml -``` -4.Change the MQTT setting in the [roomlightcase1.yaml](../../deploy/e2e) file to your own MQTT broker -```yaml - spec: - config: - broker: "tcp://192.168.8.246:1883" - password: parchk123 - username: parchk -``` -5.start the testdevice roomlight in the testdata/testdevice/roomlight directory -```shell script -cd ./testdata/testdevice/roomlight -go build -./roomlight -b "tcp://192.168.8.246:1883" -``` -6.deploy the DeviceLink use [roomlightcase1.yaml](../../deploy/e2e) -```shell script -kubeclt apply -f roomlightcase1.yaml -``` -7.check the resource status of devices in the clusters -```shell script -kubeclt get MqttDevice mqtt-test -oyaml -``` - -if all right you will get the resource info like this: -```yaml -apiVersion: "devices.edge.cattle.io/v1alpha1" -kind: "MqttDevice" -metadata: - creationTimestamp: - name: "testDevice" -spec: - config: - broker: "" - password: "" - username: "" - properties: - - description: "test property" - jsonPath: "power" - name: "test_property" - pubInfo: - qos: "0" - topic: "" - subInfo: - payloadType: "json" - qos: "2" - topic: "test/abc" - value: - valueType: "" -status: - properties: - - description: "test property" - name: "test_property" - updateAt: "2020-05-20T09:04:46Z" - value: - objectValue: - electricQuantity: "19.99" - powerDissipation: "10KWH" - valueType: "object" -``` -For example, if you want to modify the value of a device's attributes, you can modify DeviceLinke's attributes,use cmd -``` -kubectl edit dl mqtt-test -``` -and add spec property value type like this: -```yaml - spec: - config: - broker: tcp://192.168.8.246:1883 - password: parchk123 - username: parchk - properties: - - description: the room light switch - jsonPath: switch - name: switch - subInfo: - payloadType: json - qos: 2 - topic: device/room/light - value: - stringValue: "on" - valueType: string - -``` \ No newline at end of file diff --git a/adaptors/mqtt/test/testdata/testdevice/roomlight/cmd/roomlight.go b/adaptors/mqtt/test/testdata/testdevice/roomlight/cmd/roomlight.go deleted file mode 100644 index 76ff16f3..00000000 --- a/adaptors/mqtt/test/testdata/testdevice/roomlight/cmd/roomlight.go +++ /dev/null @@ -1,147 +0,0 @@ -package cmd - -import ( - "encoding/json" - "fmt" - "time" - - MQTT "github.com/eclipse/paho.mqtt.golang" - "github.com/spf13/cobra" - ctrl "sigs.k8s.io/controller-runtime" -) - -const ( - waitTimeout = time.Second * 10 -) - -var roomLight RoomLight - -func NewCommand() *cobra.Command { - var c = &cobra.Command{ - Use: "roomlight", - Long: "test device roomlight", - RunE: func(cmd *cobra.Command, args []string) error { - roomLight.Run() - return nil - }, - } - - c.Flags().StringVarP(&roomLight.cfg.Broker, "broker", "b", "tcp://127.0.0.1:1883", "broker of mqtt") - c.Flags().StringVarP(&roomLight.cfg.SubTopic, "subtopic", "s", "device/room/light/cmd", "subscription topic") - c.Flags().StringVarP(&roomLight.cfg.PubTopic, "pubtopic", "p", "device/room/light", "publish topic") - c.Flags().StringVarP(&roomLight.cfg.Username, "username", "u", "parchk", "username of mqtt broker") - c.Flags().StringVarP(&roomLight.cfg.Password, "password", "w", "test123", "password of mqtt broker") - c.Flags().IntVarP(&roomLight.cfg.Qos, "qos", "q", 0, "qos of mqtt message") - - c.MarkFlagRequired("broker") - - return c -} - -type Config struct { - SubTopic string - PubTopic string - Qos int - Broker string - Username string - Password string -} - -type PowerStatus struct { - PowerDissipation string `json:"powerDissipation"` - ElectricQuantity string `json:"electricQuantity"` -} - -type Status struct { - Switch string `json:"switch"` - Brightness int `json:"brightness"` - Power PowerStatus `json:"power"` - Attr []int `json:"attr"` -} - -type RoomLight struct { - cfg Config - Status Status - Client MQTT.Client -} - -func (r *RoomLight) Run() { - tick := time.NewTicker(time.Second * 5) - var stop = ctrl.SetupSignalHandler() - if err := r.Init(); err != nil { - fmt.Println("room light init error:", err.Error()) - return - } - if err := r.Subscribe(); err != nil { - fmt.Println("room light subscribe error:", err.Error()) - return - } - for { - select { - case <-tick.C: - if err := r.Report(); err != nil { - fmt.Println("report error:", err.Error()) - } - case <-stop: - return - } - } -} - -func (r *RoomLight) Init() error { - - r.Status.Switch = "off" - r.Status.Brightness = 1 - r.Status.Power.PowerDissipation = "10KWh" - r.Status.Power.ElectricQuantity = "10%" - r.Status.Attr = append(r.Status.Attr, 13) - r.Status.Attr = append(r.Status.Attr, 15) - - opts := MQTT.NewClientOptions() - opts.AddBroker(r.cfg.Broker) - opts.SetClientID("testRoomLight") - opts.SetUsername(r.cfg.Username) - opts.SetPassword(r.cfg.Password) - opts.SetOrderMatters(true) - opts.SetAutoReconnect(true) - opts.SetCleanSession(false) - - r.Client = MQTT.NewClient(opts) - if token := r.Client.Connect(); token.WaitTimeout(waitTimeout) && token.Error() != nil { - return token.Error() - } - - return nil -} - -func (r *RoomLight) Report() error { - payload, err := json.Marshal(&r.Status) - if err != nil { - return err - } - token := r.Client.Publish(r.cfg.PubTopic, byte(r.cfg.Qos), true, payload) - - if token.WaitTimeout(waitTimeout) && token.Error() != nil { - return token.Error() - } - - fmt.Printf("room light report topic: %s, status:%s \n", r.cfg.PubTopic, string(payload)) - - return nil -} - -func (r *RoomLight) Subscribe() error { - callback := func(client MQTT.Client, msg MQTT.Message) { - fmt.Println("Subscribe payload:", string(msg.Payload())) - if err := json.Unmarshal(msg.Payload(), &r.Status); err != nil { - fmt.Println("subscribe callback error:", err.Error()) - } - fmt.Printf("Subscribe update status:%+v\n", r.Status) - } - token := r.Client.Subscribe(r.cfg.SubTopic, byte(r.cfg.Qos), callback) - if token.WaitTimeout(waitTimeout) && token.Error() != nil { - return token.Error() - } - - return nil -} diff --git a/adaptors/mqtt/test/testdata/testdevice/roomlight/main.go b/adaptors/mqtt/test/testdata/testdevice/roomlight/main.go deleted file mode 100644 index 61933c3e..00000000 --- a/adaptors/mqtt/test/testdata/testdevice/roomlight/main.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "os" - - "github.com/rancher/octopus/adaptors/mqtt/test/testdata/testdevice/roomlight/cmd" -) - -func main() { - - c := cmd.NewCommand() - - if c.Execute() != nil { - os.Exit(1) - } -} diff --git a/go.mod b/go.mod index 9677bc70..8a37ab8f 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 github.com/tidwall/gjson v1.6.0 - github.com/tidwall/sjson v1.1.1 + github.com/tidwall/sjson v1.0.4 go.uber.org/atomic v1.4.0 go.uber.org/zap v1.10.0 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e diff --git a/go.sum b/go.sum index ef532a49..a4311b61 100644 --- a/go.sum +++ b/go.sum @@ -360,11 +360,10 @@ github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= -github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U= -github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs= +github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg= +github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= diff --git a/hack/make-rules/template-adaptor.sh b/hack/make-rules/template-adaptor.sh index e680aacb..407c9cec 100755 --- a/hack/make-rules/template-adaptor.sh +++ b/hack/make-rules/template-adaptor.sh @@ -32,6 +32,7 @@ function entry() { fi deviceNameLowercase=$(echo -n "${deviceName}" | tr '[:upper:]' '[:lower:]') sed "s#TemplateDevice#${deviceName}#g" "${adaptorPath}/api/v1alpha1/templatedevice_types.go" >"${tmpfile}" && mv "${tmpfile}" "${adaptorPath}/api/v1alpha1/templatedevice_types.go" + sed "s#template device#${deviceName} device#g" "${adaptorPath}/api/v1alpha1/templatedevice_types.go" >"${tmpfile}" && mv "${tmpfile}" "${adaptorPath}/api/v1alpha1/templatedevice_types.go" mv "${adaptorPath}/api/v1alpha1/templatedevice_types.go" "${adaptorPath}/api/v1alpha1/${deviceNameLowercase}_types.go" # change cmd template to expected @@ -50,9 +51,10 @@ function entry() { mv "${adaptorPath}/pkg/template" "${adaptorPath}/pkg/${adaptorNameLowercase}" # change deploy template to expected - mv "${adaptorPath}/deploy/manifests/crd/base/devices.edge.cattle.io_templatedevices.yaml" "${adaptorPath}/deploy/manifests/crd/base/devices.edge.cattle.io_${deviceNameLowercase}.yaml" - sed "s#devices.edge.cattle.io_templatedevices.yaml#devices.edge.cattle.io_${deviceNameLowercase}.yaml#g" "${adaptorPath}/deploy/manifests/crd/kustomization.yaml" >"${tmpfile}" && mv "${tmpfile}" "${adaptorPath}/deploy/manifests/crd/kustomization.yaml" + mv "${adaptorPath}/deploy/manifests/crd/base/devices.edge.cattle.io_templatedevices.yaml" "${adaptorPath}/deploy/manifests/crd/base/devices.edge.cattle.io_${deviceNameLowercase}s.yaml" + sed "s#devices.edge.cattle.io_templatedevices.yaml#devices.edge.cattle.io_${deviceNameLowercase}s.yaml#g" "${adaptorPath}/deploy/manifests/crd/kustomization.yaml" >"${tmpfile}" && mv "${tmpfile}" "${adaptorPath}/deploy/manifests/crd/kustomization.yaml" sed "s#octopus-adaptor-template#octopus-adaptor-${adaptorNameLowercase}#g" "${adaptorPath}/deploy/manifests/workload/daemonset.yaml" >"${tmpfile}" && mv "${tmpfile}" "${adaptorPath}/deploy/manifests/workload/daemonset.yaml" + sed "s#octopus-adaptor-template#octopus-adaptor-${adaptorNameLowercase}#g" "${adaptorPath}/deploy/manifests/overlays/default/kustomization.yaml" >"${tmpfile}" && mv "${tmpfile}" "${adaptorPath}/deploy/manifests/overlays/default/kustomization.yaml" # change Dockerfile template to expected sed "s#template#${adaptorNameLowercase}#g" "${adaptorPath}/Dockerfile" >"${tmpfile}" && mv "${tmpfile}" "${adaptorPath}/Dockerfile" diff --git a/pkg/mqtt/client.go b/pkg/mqtt/client.go index 36fae0fe..cd7ebd5a 100644 --- a/pkg/mqtt/client.go +++ b/pkg/mqtt/client.go @@ -93,8 +93,9 @@ type Client interface { // Publish publishes the message to corresponding topic. Publish(message PublishMessage) error - // Subscribe subscribes the corresponding topic. - Subscribe(handler SubscribeHandler, topics ...SubscribeTopic) error + // Subscribe subscribes the corresponding topic and handle in the same handler, + // and deals with the unsubscribe actions automatically. + Subscribe(topics []SubscribeTopic, handler SubscribeHandler) error } type client struct { @@ -136,7 +137,7 @@ func (c *client) RawClient() mqtt.Client { return c.raw } -func (c *client) Subscribe(handler SubscribeHandler, topics ...SubscribeTopic) error { +func (c *client) Subscribe(topics []SubscribeTopic, handler SubscribeHandler) error { if len(topics) == 0 { return nil } diff --git a/template/adaptor/deploy/manifests/crd/kustomization.yaml b/template/adaptor/deploy/manifests/crd/kustomization.yaml index a76e3cc9..d97d0f3b 100644 --- a/template/adaptor/deploy/manifests/crd/kustomization.yaml +++ b/template/adaptor/deploy/manifests/crd/kustomization.yaml @@ -1,6 +1,6 @@ commonAnnotations: devices.edge.cattle.io/enable: "true" # this is required - devices.edge.cattle.io/device-property: '{"name":"string","dataType":"string","value":"string","updatedAt":"date"}' # specify device property to be displayed in the UI + devices.edge.cattle.io/device-property: "" # specify device property to be displayed in the UI devices.edge.cattle.io/icon: "" # define the icon for the Edge UI devices.edge.cattle.io/description: "" # add your custom device adaptor description diff --git a/vendor/github.com/tidwall/pretty/README.md b/vendor/github.com/tidwall/pretty/README.md index 09884692..d2b8864d 100644 --- a/vendor/github.com/tidwall/pretty/README.md +++ b/vendor/github.com/tidwall/pretty/README.md @@ -1,7 +1,7 @@ # Pretty [![Build Status](https://img.shields.io/travis/tidwall/pretty.svg?style=flat-square)](https://travis-ci.org/tidwall/prettty) [![Coverage Status](https://img.shields.io/badge/coverage-100%25-brightgreen.svg?style=flat-square)](http://gocover.io/github.com/tidwall/pretty) -[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/tidwall/pretty) +[![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://godoc.org/github.com/tidwall/pretty) Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads. diff --git a/vendor/github.com/tidwall/pretty/pretty.go b/vendor/github.com/tidwall/pretty/pretty.go index 2951c610..0a922d03 100644 --- a/vendor/github.com/tidwall/pretty/pretty.go +++ b/vendor/github.com/tidwall/pretty/pretty.go @@ -318,25 +318,21 @@ func hexp(p byte) byte { } // TerminalStyle is for terminals -var TerminalStyle *Style - -func init() { - TerminalStyle = &Style{ - Key: [2]string{"\x1B[94m", "\x1B[0m"}, - String: [2]string{"\x1B[92m", "\x1B[0m"}, - Number: [2]string{"\x1B[93m", "\x1B[0m"}, - True: [2]string{"\x1B[96m", "\x1B[0m"}, - False: [2]string{"\x1B[96m", "\x1B[0m"}, - Null: [2]string{"\x1B[91m", "\x1B[0m"}, - Append: func(dst []byte, c byte) []byte { - if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') { - dst = append(dst, "\\u00"...) - dst = append(dst, hexp((c>>4)&0xF)) - return append(dst, hexp((c)&0xF)) - } - return append(dst, c) - }, - } +var TerminalStyle = &Style{ + Key: [2]string{"\x1B[94m", "\x1B[0m"}, + String: [2]string{"\x1B[92m", "\x1B[0m"}, + Number: [2]string{"\x1B[93m", "\x1B[0m"}, + True: [2]string{"\x1B[96m", "\x1B[0m"}, + False: [2]string{"\x1B[96m", "\x1B[0m"}, + Null: [2]string{"\x1B[91m", "\x1B[0m"}, + Append: func(dst []byte, c byte) []byte { + if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') { + dst = append(dst, "\\u00"...) + dst = append(dst, hexp((c>>4)&0xF)) + return append(dst, hexp((c)&0xF)) + } + return append(dst, c) + }, } // Color will colorize the json. The style parma is used for customizing diff --git a/vendor/github.com/tidwall/sjson/go.mod b/vendor/github.com/tidwall/sjson/go.mod deleted file mode 100644 index 5ce2195d..00000000 --- a/vendor/github.com/tidwall/sjson/go.mod +++ /dev/null @@ -1,8 +0,0 @@ -module github.com/tidwall/sjson - -go 1.14 - -require ( - github.com/tidwall/gjson v1.6.0 - github.com/tidwall/pretty v1.0.1 -) diff --git a/vendor/github.com/tidwall/sjson/go.sum b/vendor/github.com/tidwall/sjson/go.sum deleted file mode 100644 index 98af3635..00000000 --- a/vendor/github.com/tidwall/sjson/go.sum +++ /dev/null @@ -1,7 +0,0 @@ -github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= -github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= -github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= -github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= diff --git a/vendor/github.com/tidwall/sjson/sjson.go b/vendor/github.com/tidwall/sjson/sjson.go index 3c1ae116..9f2d5c88 100644 --- a/vendor/github.com/tidwall/sjson/sjson.go +++ b/vendor/github.com/tidwall/sjson/sjson.go @@ -3,9 +3,7 @@ package sjson import ( jsongo "encoding/json" - "reflect" "strconv" - "unsafe" "github.com/tidwall/gjson" ) @@ -341,18 +339,12 @@ func appendRawPaths(buf []byte, jstr string, paths []pathResult, raw string, default: return nil, &errorType{"json must be an object or array"} case '{': - end := len(jsres.Raw) - 1 - for ; end > 0; end-- { - if jsres.Raw[end] == '}' { - break - } - } - buf = append(buf, jsres.Raw[:end]...) + buf = append(buf, '{') + buf = appendBuild(buf, false, paths, raw, stringify) if comma { buf = append(buf, ',') } - buf = appendBuild(buf, false, paths, raw, stringify) - buf = append(buf, '}') + buf = append(buf, jsres.Raw[1:]...) return buf, nil case '[': var appendit bool @@ -488,182 +480,3 @@ func Delete(json, path string) (string, error) { func DeleteBytes(json []byte, path string) ([]byte, error) { return SetBytes(json, path, dtype{}) } - -func set(jstr, path, raw string, - stringify, del, optimistic, inplace bool) ([]byte, error) { - if path == "" { - return nil, &errorType{"path cannot be empty"} - } - if !del && optimistic && isOptimisticPath(path) { - res := gjson.Get(jstr, path) - if res.Exists() && res.Index > 0 { - sz := len(jstr) - len(res.Raw) + len(raw) - if stringify { - sz += 2 - } - if inplace && sz <= len(jstr) { - if !stringify || !mustMarshalString(raw) { - jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&jstr)) - jsonbh := reflect.SliceHeader{ - Data: jsonh.Data, Len: jsonh.Len, Cap: jsonh.Len} - jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh)) - if stringify { - jbytes[res.Index] = '"' - copy(jbytes[res.Index+1:], []byte(raw)) - jbytes[res.Index+1+len(raw)] = '"' - copy(jbytes[res.Index+1+len(raw)+1:], - jbytes[res.Index+len(res.Raw):]) - } else { - copy(jbytes[res.Index:], []byte(raw)) - copy(jbytes[res.Index+len(raw):], - jbytes[res.Index+len(res.Raw):]) - } - return jbytes[:sz], nil - } - return nil, nil - } - buf := make([]byte, 0, sz) - buf = append(buf, jstr[:res.Index]...) - if stringify { - buf = appendStringify(buf, raw) - } else { - buf = append(buf, raw...) - } - buf = append(buf, jstr[res.Index+len(res.Raw):]...) - return buf, nil - } - } - // parse the path, make sure that it does not contain invalid characters - // such as '#', '?', '*' - paths := make([]pathResult, 0, 4) - r, err := parsePath(path) - if err != nil { - return nil, err - } - paths = append(paths, r) - for r.more { - if r, err = parsePath(r.path); err != nil { - return nil, err - } - paths = append(paths, r) - } - - njson, err := appendRawPaths(nil, jstr, paths, raw, stringify, del) - if err != nil { - return nil, err - } - return njson, nil -} - -// SetOptions sets a json value for the specified path with options. -// A path is in dot syntax, such as "name.last" or "age". -// This function expects that the json is well-formed, and does not validate. -// Invalid json will not panic, but it may return back unexpected results. -// An error is returned if the path is not valid. -func SetOptions(json, path string, value interface{}, - opts *Options) (string, error) { - if opts != nil { - if opts.ReplaceInPlace { - // it's not safe to replace bytes in-place for strings - // copy the Options and set options.ReplaceInPlace to false. - nopts := *opts - opts = &nopts - opts.ReplaceInPlace = false - } - } - jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&json)) - jsonbh := reflect.SliceHeader{Data: jsonh.Data, Len: jsonh.Len} - jsonb := *(*[]byte)(unsafe.Pointer(&jsonbh)) - res, err := SetBytesOptions(jsonb, path, value, opts) - return string(res), err -} - -// SetBytesOptions sets a json value for the specified path with options. -// If working with bytes, this method preferred over -// SetOptions(string(data), path, value) -func SetBytesOptions(json []byte, path string, value interface{}, - opts *Options) ([]byte, error) { - var optimistic, inplace bool - if opts != nil { - optimistic = opts.Optimistic - inplace = opts.ReplaceInPlace - } - jstr := *(*string)(unsafe.Pointer(&json)) - var res []byte - var err error - switch v := value.(type) { - default: - b, merr := jsongo.Marshal(value) - if merr != nil { - return nil, merr - } - raw := *(*string)(unsafe.Pointer(&b)) - res, err = set(jstr, path, raw, false, false, optimistic, inplace) - case dtype: - res, err = set(jstr, path, "", false, true, optimistic, inplace) - case string: - res, err = set(jstr, path, v, true, false, optimistic, inplace) - case []byte: - raw := *(*string)(unsafe.Pointer(&v)) - res, err = set(jstr, path, raw, true, false, optimistic, inplace) - case bool: - if v { - res, err = set(jstr, path, "true", false, false, optimistic, inplace) - } else { - res, err = set(jstr, path, "false", false, false, optimistic, inplace) - } - case int8: - res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), - false, false, optimistic, inplace) - case int16: - res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), - false, false, optimistic, inplace) - case int32: - res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), - false, false, optimistic, inplace) - case int64: - res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), - false, false, optimistic, inplace) - case uint8: - res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), - false, false, optimistic, inplace) - case uint16: - res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), - false, false, optimistic, inplace) - case uint32: - res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), - false, false, optimistic, inplace) - case uint64: - res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), - false, false, optimistic, inplace) - case float32: - res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), - false, false, optimistic, inplace) - case float64: - res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), - false, false, optimistic, inplace) - } - if err == errNoChange { - return json, nil - } - return res, err -} - -// SetRawBytesOptions sets a raw json value for the specified path with options. -// If working with bytes, this method preferred over -// SetRawOptions(string(data), path, value, opts) -func SetRawBytesOptions(json []byte, path string, value []byte, - opts *Options) ([]byte, error) { - jstr := *(*string)(unsafe.Pointer(&json)) - vstr := *(*string)(unsafe.Pointer(&value)) - var optimistic, inplace bool - if opts != nil { - optimistic = opts.Optimistic - inplace = opts.ReplaceInPlace - } - res, err := set(jstr, path, vstr, false, false, optimistic, inplace) - if err == errNoChange { - return json, nil - } - return res, err -} diff --git a/vendor/github.com/tidwall/sjson/sjson_gae.go b/vendor/github.com/tidwall/sjson/sjson_gae.go new file mode 100644 index 00000000..eaba00e8 --- /dev/null +++ b/vendor/github.com/tidwall/sjson/sjson_gae.go @@ -0,0 +1,196 @@ +//+build appengine + +package sjson + +import ( + jsongo "encoding/json" + "strconv" + + "github.com/tidwall/gjson" +) + +func set(jstr, path, raw string, + stringify, del, optimistic, inplace bool) ([]byte, error) { + if path == "" { + return nil, &errorType{"path cannot be empty"} + } + if !del && optimistic && isOptimisticPath(path) { + res := gjson.Get(jstr, path) + if res.Exists() && res.Index > 0 { + sz := len(jstr) - len(res.Raw) + len(raw) + if stringify { + sz += 2 + } + if inplace && sz <= len(jstr) { + if !stringify || !mustMarshalString(raw) { + // jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&jstr)) + // jsonbh := reflect.SliceHeader{ + // Data: jsonh.Data, Len: jsonh.Len, Cap: jsonh.Len} + // jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh)) + jbytes := []byte(jstr) + if stringify { + jbytes[res.Index] = '"' + copy(jbytes[res.Index+1:], []byte(raw)) + jbytes[res.Index+1+len(raw)] = '"' + copy(jbytes[res.Index+1+len(raw)+1:], + jbytes[res.Index+len(res.Raw):]) + } else { + copy(jbytes[res.Index:], []byte(raw)) + copy(jbytes[res.Index+len(raw):], + jbytes[res.Index+len(res.Raw):]) + } + return jbytes[:sz], nil + } + return nil, nil + } + buf := make([]byte, 0, sz) + buf = append(buf, jstr[:res.Index]...) + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + buf = append(buf, jstr[res.Index+len(res.Raw):]...) + return buf, nil + } + } + // parse the path, make sure that it does not contain invalid characters + // such as '#', '?', '*' + paths := make([]pathResult, 0, 4) + r, err := parsePath(path) + if err != nil { + return nil, err + } + paths = append(paths, r) + for r.more { + if r, err = parsePath(r.path); err != nil { + return nil, err + } + paths = append(paths, r) + } + + njson, err := appendRawPaths(nil, jstr, paths, raw, stringify, del) + if err != nil { + return nil, err + } + return njson, nil +} + +// SetOptions sets a json value for the specified path with options. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// An error is returned if the path is not valid. +func SetOptions(json, path string, value interface{}, + opts *Options) (string, error) { + if opts != nil { + if opts.ReplaceInPlace { + // it's not safe to replace bytes in-place for strings + // copy the Options and set options.ReplaceInPlace to false. + nopts := *opts + opts = &nopts + opts.ReplaceInPlace = false + } + } + // jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&json)) + // jsonbh := reflect.SliceHeader{Data: jsonh.Data, Len: jsonh.Len} + // jsonb := *(*[]byte)(unsafe.Pointer(&jsonbh)) + jsonb := []byte(json) + res, err := SetBytesOptions(jsonb, path, value, opts) + return string(res), err +} + +// SetBytesOptions sets a json value for the specified path with options. +// If working with bytes, this method preferred over +// SetOptions(string(data), path, value) +func SetBytesOptions(json []byte, path string, value interface{}, + opts *Options) ([]byte, error) { + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + // jstr := *(*string)(unsafe.Pointer(&json)) + jstr := string(json) + var res []byte + var err error + switch v := value.(type) { + default: + b, merr := jsongo.Marshal(value) + if merr != nil { + return nil, merr + } + // raw := *(*string)(unsafe.Pointer(&b)) + raw := string(b) + res, err = set(jstr, path, raw, false, false, optimistic, inplace) + case dtype: + res, err = set(jstr, path, "", false, true, optimistic, inplace) + case string: + res, err = set(jstr, path, v, true, false, optimistic, inplace) + case []byte: + // raw := *(*string)(unsafe.Pointer(&v)) + raw := string(v) + res, err = set(jstr, path, raw, true, false, optimistic, inplace) + case bool: + if v { + res, err = set(jstr, path, "true", false, false, optimistic, inplace) + } else { + res, err = set(jstr, path, "false", false, false, optimistic, inplace) + } + case int8: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int16: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int32: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int64: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case uint8: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint16: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint32: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint64: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case float32: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + case float64: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + } + if err == errNoChange { + return json, nil + } + return res, err +} + +// SetRawBytesOptions sets a raw json value for the specified path with options. +// If working with bytes, this method preferred over +// SetRawOptions(string(data), path, value, opts) +func SetRawBytesOptions(json []byte, path string, value []byte, + opts *Options) ([]byte, error) { + // jstr := *(*string)(unsafe.Pointer(&json)) + // vstr := *(*string)(unsafe.Pointer(&value)) + jstr := string(json) + vstr := string(value) + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + res, err := set(jstr, path, vstr, false, false, optimistic, inplace) + if err == errNoChange { + return json, nil + } + return res, err +} diff --git a/vendor/github.com/tidwall/sjson/sjson_ngae.go b/vendor/github.com/tidwall/sjson/sjson_ngae.go new file mode 100644 index 00000000..6f47a047 --- /dev/null +++ b/vendor/github.com/tidwall/sjson/sjson_ngae.go @@ -0,0 +1,191 @@ +//+build !appengine + +package sjson + +import ( + jsongo "encoding/json" + "reflect" + "strconv" + "unsafe" + + "github.com/tidwall/gjson" +) + +func set(jstr, path, raw string, + stringify, del, optimistic, inplace bool) ([]byte, error) { + if path == "" { + return nil, &errorType{"path cannot be empty"} + } + if !del && optimistic && isOptimisticPath(path) { + res := gjson.Get(jstr, path) + if res.Exists() && res.Index > 0 { + sz := len(jstr) - len(res.Raw) + len(raw) + if stringify { + sz += 2 + } + if inplace && sz <= len(jstr) { + if !stringify || !mustMarshalString(raw) { + jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&jstr)) + jsonbh := reflect.SliceHeader{ + Data: jsonh.Data, Len: jsonh.Len, Cap: jsonh.Len} + jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh)) + if stringify { + jbytes[res.Index] = '"' + copy(jbytes[res.Index+1:], []byte(raw)) + jbytes[res.Index+1+len(raw)] = '"' + copy(jbytes[res.Index+1+len(raw)+1:], + jbytes[res.Index+len(res.Raw):]) + } else { + copy(jbytes[res.Index:], []byte(raw)) + copy(jbytes[res.Index+len(raw):], + jbytes[res.Index+len(res.Raw):]) + } + return jbytes[:sz], nil + } + return nil, nil + } + buf := make([]byte, 0, sz) + buf = append(buf, jstr[:res.Index]...) + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + buf = append(buf, jstr[res.Index+len(res.Raw):]...) + return buf, nil + } + } + // parse the path, make sure that it does not contain invalid characters + // such as '#', '?', '*' + paths := make([]pathResult, 0, 4) + r, err := parsePath(path) + if err != nil { + return nil, err + } + paths = append(paths, r) + for r.more { + if r, err = parsePath(r.path); err != nil { + return nil, err + } + paths = append(paths, r) + } + + njson, err := appendRawPaths(nil, jstr, paths, raw, stringify, del) + if err != nil { + return nil, err + } + return njson, nil +} + +// SetOptions sets a json value for the specified path with options. +// A path is in dot syntax, such as "name.last" or "age". +// This function expects that the json is well-formed, and does not validate. +// Invalid json will not panic, but it may return back unexpected results. +// An error is returned if the path is not valid. +func SetOptions(json, path string, value interface{}, + opts *Options) (string, error) { + if opts != nil { + if opts.ReplaceInPlace { + // it's not safe to replace bytes in-place for strings + // copy the Options and set options.ReplaceInPlace to false. + nopts := *opts + opts = &nopts + opts.ReplaceInPlace = false + } + } + jsonh := *(*reflect.StringHeader)(unsafe.Pointer(&json)) + jsonbh := reflect.SliceHeader{Data: jsonh.Data, Len: jsonh.Len} + jsonb := *(*[]byte)(unsafe.Pointer(&jsonbh)) + res, err := SetBytesOptions(jsonb, path, value, opts) + return string(res), err +} + +// SetBytesOptions sets a json value for the specified path with options. +// If working with bytes, this method preferred over +// SetOptions(string(data), path, value) +func SetBytesOptions(json []byte, path string, value interface{}, + opts *Options) ([]byte, error) { + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + jstr := *(*string)(unsafe.Pointer(&json)) + var res []byte + var err error + switch v := value.(type) { + default: + b, merr := jsongo.Marshal(value) + if merr != nil { + return nil, merr + } + raw := *(*string)(unsafe.Pointer(&b)) + res, err = set(jstr, path, raw, false, false, optimistic, inplace) + case dtype: + res, err = set(jstr, path, "", false, true, optimistic, inplace) + case string: + res, err = set(jstr, path, v, true, false, optimistic, inplace) + case []byte: + raw := *(*string)(unsafe.Pointer(&v)) + res, err = set(jstr, path, raw, true, false, optimistic, inplace) + case bool: + if v { + res, err = set(jstr, path, "true", false, false, optimistic, inplace) + } else { + res, err = set(jstr, path, "false", false, false, optimistic, inplace) + } + case int8: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int16: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int32: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case int64: + res, err = set(jstr, path, strconv.FormatInt(int64(v), 10), + false, false, optimistic, inplace) + case uint8: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint16: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint32: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case uint64: + res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10), + false, false, optimistic, inplace) + case float32: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + case float64: + res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64), + false, false, optimistic, inplace) + } + if err == errNoChange { + return json, nil + } + return res, err +} + +// SetRawBytesOptions sets a raw json value for the specified path with options. +// If working with bytes, this method preferred over +// SetRawOptions(string(data), path, value, opts) +func SetRawBytesOptions(json []byte, path string, value []byte, + opts *Options) ([]byte, error) { + jstr := *(*string)(unsafe.Pointer(&json)) + vstr := *(*string)(unsafe.Pointer(&value)) + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + res, err := set(jstr, path, vstr, false, false, optimistic, inplace) + if err == errNoChange { + return json, nil + } + return res, err +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 84c6f1b9..79fefc9e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -178,9 +178,9 @@ github.com/stretchr/testify/assert github.com/tidwall/gjson # github.com/tidwall/match v1.0.1 github.com/tidwall/match -# github.com/tidwall/pretty v1.0.1 +# github.com/tidwall/pretty v1.0.0 github.com/tidwall/pretty -# github.com/tidwall/sjson v1.1.1 +# github.com/tidwall/sjson v1.0.4 github.com/tidwall/sjson # go.uber.org/atomic v1.4.0 go.uber.org/atomic