diff --git a/README.md b/README.md
index 02d67a6..3bd05f4 100644
--- a/README.md
+++ b/README.md
@@ -33,6 +33,8 @@ The default file path is `.github/labels.yml`, but you can specify any file path
 
 To create manifest of the current labels easily, using [label-exporter](https://github.com/micnncim/label-exporter) is recommended.
 
+Color code can be defined in either format `d73a4a` or `'#d73a4a'` (note the quotes).
+
 ### Create Workflow
 
 An example workflow is here.
diff --git a/go.mod b/go.mod
index 61ac80c..b498e80 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.13
 require (
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/stretchr/testify v1.7.0
 	go.uber.org/multierr v1.7.0
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
diff --git a/pkg/github/github.go b/pkg/github/github.go
index 361a450..a80241e 100644
--- a/pkg/github/github.go
+++ b/pkg/github/github.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"strings"
 
 	"github.com/google/go-github/github"
 	"golang.org/x/oauth2"
@@ -41,9 +42,19 @@ func FromManifestToLabels(path string) ([]Label, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	var labels []Label
-	err = yaml.Unmarshal(buf, &labels)
-	return labels, err
+
+	if err := yaml.Unmarshal(buf, &labels); err != nil {
+		return labels, err
+	}
+
+	for i, label := range labels {
+		label.Color = strings.TrimPrefix(label.Color, "#")
+		labels[i] = label
+	}
+
+	return labels, nil
 }
 
 func NewClient(token string) *Client {
diff --git a/pkg/github/github_test.go b/pkg/github/github_test.go
new file mode 100644
index 0000000..5315821
--- /dev/null
+++ b/pkg/github/github_test.go
@@ -0,0 +1,32 @@
+package github_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/micnncim/action-label-syncer/pkg/github"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFromManifestToLabels(t *testing.T) {
+	labels, err := github.FromManifestToLabels(filepath.Join("testdata", "labels.yml"))
+	require.NoError(t, err)
+	assert.ElementsMatch(t, labels, []github.Label{
+		{
+			Name:        "bug",
+			Description: "Something isn't working",
+			Color:       "d73a4a",
+		},
+		{
+			Name:        "documentation",
+			Description: "Improvements or additions to documentation",
+			Color:       "0075ca",
+		},
+		{
+			Name:        "duplicate",
+			Description: "This issue or pull request already exists",
+			Color:       "cfd3d7",
+		},
+	})
+}
diff --git a/pkg/github/testdata/labels.yml b/pkg/github/testdata/labels.yml
new file mode 100644
index 0000000..a2c3a97
--- /dev/null
+++ b/pkg/github/testdata/labels.yml
@@ -0,0 +1,9 @@
+- name: bug
+  description: Something isn't working
+  color: d73a4a
+- name: documentation
+  description: Improvements or additions to documentation
+  color: '#0075ca'
+- name: duplicate
+  description: This issue or pull request already exists
+  color: cfd3d7